From c2b82fb2dd862aed94305e2292bf650b54032ae8 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Mon, 27 Oct 2025 09:51:09 -0400 Subject: [PATCH] Use pyproject and uv Also, switch from using from `pkg_resources.iter_entry_points` to `importlib.metadata`. Since is removed in Python 3.12. I had a report that this was causing a problem. I guess people are still using twarc, I suppose with keys that they pay for... --- .github/workflows/main.yml | 24 ++++----------- .gitignore | 1 + README.md | 9 ++++-- pyproject.toml | 43 +++++++++++++++++++++++++++ requirements.txt | 7 ----- setup.py | 45 ----------------------------- {twarc => src/twarc}/__init__.py | 0 {twarc => src/twarc}/__main__.py | 0 {twarc => src/twarc}/client.py | 0 {twarc => src/twarc}/client2.py | 0 {twarc => src/twarc}/command.py | 0 {twarc => src/twarc}/command2.py | 19 ++++++------ {twarc => src/twarc}/config.py | 0 {twarc => src/twarc}/decorators.py | 0 {twarc => src/twarc}/decorators2.py | 0 {twarc => src/twarc}/expansions.py | 0 {twarc => src/twarc}/handshake.py | 0 {twarc => src/twarc}/json2csv.py | 0 {twarc => src/twarc}/version.py | 2 +- 19 files changed, 65 insertions(+), 85 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py rename {twarc => src/twarc}/__init__.py (100%) rename {twarc => src/twarc}/__main__.py (100%) rename {twarc => src/twarc}/client.py (100%) rename {twarc => src/twarc}/client2.py (100%) rename {twarc => src/twarc}/command.py (100%) rename {twarc => src/twarc}/command2.py (99%) rename {twarc => src/twarc}/config.py (100%) rename {twarc => src/twarc}/decorators.py (100%) rename {twarc => src/twarc}/decorators2.py (100%) rename {twarc => src/twarc}/expansions.py (100%) rename {twarc => src/twarc}/handshake.py (100%) rename {twarc => src/twarc}/json2csv.py (100%) rename {twarc => src/twarc}/version.py (89%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0aa5070..6a047c94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.12] steps: - uses: actions/checkout@v2 @@ -25,10 +25,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + - name: Install uv + uses: astral-sh/setup-uv@v6 - name: Test with pytest env: @@ -38,12 +36,11 @@ jobs: ACCESS_TOKEN_SECRET: ${{ secrets.access_token_secret }} BEARER_TOKEN: ${{ secrets.bearer_token }} SKIP_ACADEMIC_PRODUCT_TRACK: true - run: python setup.py test + run: uv run pytest - name: Ensure packages can be built run: | - python -m pip install wheel - python setup.py sdist bdist_wheel + uv build - name: Publish to PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') @@ -51,14 +48,3 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - - - name: Slack Notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - SLACK_USERNAME: docnow-build - SLACK_ICON_EMOJI: ":test-tube:" - SLACK_COLOR: "${{ job.status == 'success' && 'good' || 'danger' }}" - SLACK_MESSAGE: "Tests Results: ${{ job.status }}" - - diff --git a/.gitignore b/.gitignore index 1b6c7432..3989fdaa 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ twarc.egg-info .vscode .env site +uv.lock diff --git a/README.md b/README.md index b4312d7d..b7d35916 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # twarc -[![DOI](https://zenodo.org/badge/7605723.svg)](https://zenodo.org/badge/latestdoi/7605723) [![Build Status](https://github.com/docnow/twarc/workflows/tests/badge.svg)](https://github.com/DocNow/twarc/actions/workflows/main.yml) [![Standard](https://img.shields.io/endpoint?url=https%3A%2F%2Ftwbadges.glitch.me%2Fbadges%2Fstandard)](https://developer.twitter.com/en/docs/twitter-api) [![Premium](https://img.shields.io/endpoint?url=https%3A%2F%2Ftwbadges.glitch.me%2Fbadges%2Fpremium)](https://developer.twitter.com/) [![v2](https://img.shields.io/endpoint?url=https%3A%2F%2Ftwbadges.glitch.me%2Fbadges%2Fv2)](https://developer.twitter.com/en/docs/twitter-api) +**Note: twarc is no longer actively supported after changes to Twitter's API quotas made it unusable.** + +--- + +[![DOI](https://zenodo.org/badge/7605723.svg)](https://zenodo.org/badge/latestdoi/7605723) twarc is a command line tool and Python library for collecting and archiving Twitter JSON data via the Twitter API. It has separate commands (twarc and twarc2) for working with the older @@ -41,7 +45,6 @@ If you are interested in adding functionality to twarc or fixing something that' git clone https://github.com/docnow/twarc cd twarc - pip install -r requirements.txt Create a .env file that included Twitter App keys to use during testing: @@ -53,7 +56,7 @@ Create a .env file that included Twitter App keys to use during testing: Now run the tests: - python setup.py test + uv run pytest Add your code and some new tests, and send a pull request! diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..765ffd10 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name = "twarc" +version = "2.14.1" +description = "Archive tweets from the command line" +license = "MIT" +readme = "README.md" +requires-python = ">=3.13" +dependencies = [ + "click>=7,<9", + "click-config-file>=0.6", + "click-plugins>=1", + "humanize>=3.9", + "python-dateutil>=2.8", + "requests_oauthlib>=1.3", + "tqdm>=4.62", + "twarc-csv>=0.7.2", +] + +[dependency-groups] +dev = [ + "black>=25.9.0", + "pytest>=8.4.2", + "pytest-black>=0.6.0", + "python-dotenv>=1.2.1", + "pytz>=2025.2", + "toml>=0.10.2", +] + +[project.scripts] +twarc = "twarc.command:main" +twarc2 = "twarc.command2:twarc2" + +[tool.pytest.ini_options] +addopts = "--verbose --black" + +[tool.uv.workspace] +members = [ + "tmp/twarc", +] + +[build-system] +requires = ["uv_build>=0.8.3,<0.9.0"] +build-backend = "uv_build" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9a7c9f06..00000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -click>=7,<9 -click-config-file>=0.6 -click-plugins>=1 -humanize>=3.9 -python-dateutil>=2.8 -requests_oauthlib>=1.3 -tqdm>=4.62 diff --git a/setup.py b/setup.py deleted file mode 100644 index 0d96c2c6..00000000 --- a/setup.py +++ /dev/null @@ -1,45 +0,0 @@ -import sys -import setuptools - -with open("twarc/version.py") as f: - exec(f.read()) - -with open("docs/README.md") as f: - long_description = f.read() - -with open("requirements.txt") as f: - dependencies = f.read().split() - -if __name__ == "__main__": - setuptools.setup( - name="twarc", - version=version, - url="https://github.com/docnow/twarc", - author="Ed Summers", - author_email="ehs@pobox.com", - packages=["twarc"], - description="Archive tweets from the command line", - long_description=long_description, - long_description_content_type="text/markdown", - license="MIT", - classifiers=[ - "License :: OSI Approved :: MIT License", - ], - python_requires=">=3.6", - install_requires=dependencies, - setup_requires=["pytest-runner"], - tests_require=[ - "pytest", - "black", - "pytest-black", - "python-dotenv", - "pytz", - "tomli", - ], - entry_points={ - "console_scripts": [ - "twarc = twarc.command:main", - "twarc2 = twarc.command2:twarc2", - ] - }, - ) diff --git a/twarc/__init__.py b/src/twarc/__init__.py similarity index 100% rename from twarc/__init__.py rename to src/twarc/__init__.py diff --git a/twarc/__main__.py b/src/twarc/__main__.py similarity index 100% rename from twarc/__main__.py rename to src/twarc/__main__.py diff --git a/twarc/client.py b/src/twarc/client.py similarity index 100% rename from twarc/client.py rename to src/twarc/client.py diff --git a/twarc/client2.py b/src/twarc/client2.py similarity index 100% rename from twarc/client2.py rename to src/twarc/client2.py diff --git a/twarc/command.py b/src/twarc/command.py similarity index 100% rename from twarc/command.py rename to src/twarc/command.py diff --git a/twarc/command2.py b/src/twarc/command2.py similarity index 99% rename from twarc/command2.py rename to src/twarc/command2.py index e2be2f3b..23ad66dd 100644 --- a/twarc/command2.py +++ b/src/twarc/command2.py @@ -2,7 +2,6 @@ The command line interfact to the Twitter v2 API. """ -from itertools import filterfalse import os import re import json @@ -22,7 +21,7 @@ from datetime import timezone from click_plugins import with_plugins -from pkg_resources import iter_entry_points +from importlib.metadata import entry_points from twarc.version import version from twarc.handshake import handshake @@ -37,7 +36,7 @@ PLACE_FIELDS, LIST_FIELDS, ) -from click import command, option, Option, UsageError +from click import Option, UsageError from click_config_file import configuration_option from twarc.decorators2 import ( cli_api_error, @@ -53,7 +52,7 @@ log = logging.getLogger("twarc") -@with_plugins(iter_entry_points("twarc.plugins")) +@with_plugins(entry_points(group="twarc.plugins")) @click.group() @click.option( "--consumer-key", @@ -788,7 +787,7 @@ def tweet(T, tweet_id, outfile, pretty, **kwargs): if "https" in tweet_id: tweet_id = tweet_id.split("/")[-1] - if not re.match("^\d+$", tweet_id): + if not re.match(r"^\d+$", tweet_id): click.echo(click.style("Please enter a tweet URL or ID", fg="red"), err=True) result = next(T.tweet_lookup([tweet_id], **kwargs)) _write(result, outfile, pretty=pretty) @@ -909,7 +908,7 @@ def liking_users(T, tweet_id, outfile, limit, max_results, hide_progress): """ lookup_total = 1 - if not re.match("^\d+$", str(tweet_id)): + if not re.match(r"^\d+$", str(tweet_id)): click.echo(click.style("Please enter a tweet ID", fg="red"), err=True) hide_progress = True if (outfile.name == "") else hide_progress @@ -959,7 +958,7 @@ def retweeted_by(T, tweet_id, outfile, limit, max_results, hide_progress): """ lookup_total = 0 - if not re.match("^\d+$", str(tweet_id)): + if not re.match(r"^\d+$", str(tweet_id)): click.echo(click.style("Please enter a tweet ID", fg="red"), err=True) hide_progress = True if (outfile.name == "") else hide_progress @@ -1016,7 +1015,7 @@ def quotes(T, tweet_id, outfile, limit, max_results, hide_progress, **kwargs): kwargs.pop("poll_fields", None) kwargs.pop("place_fields", None) - if not re.match("^\d+$", str(tweet_id)): + if not re.match(r"^\d+$", str(tweet_id)): click.echo(click.style("Please enter a tweet ID", fg="red"), err=True) hide_progress = True if (outfile.name == "") else hide_progress @@ -2065,7 +2064,7 @@ def lists_lookup(T, list_id, outfile, pretty, **kwargs): if "https" in list_id: list_id = list_id.split("/")[-1] - if not re.match("^\d+$", list_id): + if not re.match(r"^\d+$", list_id): click.echo(click.style("Please enter a List URL or ID", fg="red"), err=True) result = T.list_lookup(list_id, **kwargs) _write(result, outfile, pretty=pretty) @@ -2097,7 +2096,7 @@ def lists_bulk_lookup(T, infile, outfile, hide_progress, **kwargs): if "https" in list_id: list_id = list_id.split("/")[-1] - if not re.match("^\d+$", list_id): + if not re.match(r"^\d+$", list_id): click.echo( click.style("Skipping invalid List URL or ID: {line}", fg="red"), err=True, diff --git a/twarc/config.py b/src/twarc/config.py similarity index 100% rename from twarc/config.py rename to src/twarc/config.py diff --git a/twarc/decorators.py b/src/twarc/decorators.py similarity index 100% rename from twarc/decorators.py rename to src/twarc/decorators.py diff --git a/twarc/decorators2.py b/src/twarc/decorators2.py similarity index 100% rename from twarc/decorators2.py rename to src/twarc/decorators2.py diff --git a/twarc/expansions.py b/src/twarc/expansions.py similarity index 100% rename from twarc/expansions.py rename to src/twarc/expansions.py diff --git a/twarc/handshake.py b/src/twarc/handshake.py similarity index 100% rename from twarc/handshake.py rename to src/twarc/handshake.py diff --git a/twarc/json2csv.py b/src/twarc/json2csv.py similarity index 100% rename from twarc/json2csv.py rename to src/twarc/json2csv.py diff --git a/twarc/version.py b/src/twarc/version.py similarity index 89% rename from twarc/version.py rename to src/twarc/version.py index 41dcf17a..947c19e2 100644 --- a/twarc/version.py +++ b/src/twarc/version.py @@ -1,5 +1,5 @@ import platform -version = "2.14.0" +version = "2.14.1" user_agent = f"twarc/{version} ({platform.system()} {platform.machine()}) {platform.python_implementation()}/{platform.python_version()}"