From d4b25da4cd681e6f17365c3eb47a8f9004a16b28 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Wed, 12 Aug 2026 23:53:29 +0100 Subject: [PATCH 1/4] feat(ENG-56): add --dsc-file to `push deb` to auto-derive sources/changes files Adds a deb-only --dsc-file option that parses a Debian .dsc control file's Files:/Checksums-Sha256: stanza and auto-fills --sources-file/--changes-file, resolving referenced filenames relative to the .dsc's directory (closes GitHub issue #56). This is layered on top of create_push_handlers()'s generic, API-model-driven option loop rather than adding per-format branching to it: the deb subcommand alone gets the extra option, and a small resolver module (cloudsmith_cli/cli/dsc_parser.py) does the parsing with the stdlib email parser (no new dependency). Precedence: explicit --sources-file/--changes-file always win over values derived from --dsc-file, so passing both is a deliberate override rather than an error. A .dsc referencing a multi-component source package (*.orig-*.tar.*) or a detached signature (*.asc) raises a clear click.UsageError before any network call, since Cloudsmith's deb upload format has no field for either. Tests cover the parser directly (happy path with/without a .changes file, the Checksums-Sha256 fallback, missing referenced files, multi-component/detached-signature rejection, ambiguous multi-tarball .dsc files) and the CLI integration (flag precedence for both --sources-file and --changes-file, --help wiring restricted to `deb`, and that rejected .dsc files abort before validate_create_package is ever called). Co-Authored-By: Claude Sonnet 5 --- cloudsmith_cli/cli/commands/push.py | 43 +++++ cloudsmith_cli/cli/dsc_parser.py | 169 ++++++++++++++++ cloudsmith_cli/cli/tests/test_dsc_parser.py | 122 ++++++++++++ cloudsmith_cli/cli/tests/test_push_dsc_cli.py | 181 ++++++++++++++++++ 4 files changed, 515 insertions(+) create mode 100644 cloudsmith_cli/cli/dsc_parser.py create mode 100644 cloudsmith_cli/cli/tests/test_dsc_parser.py create mode 100644 cloudsmith_cli/cli/tests/test_push_dsc_cli.py diff --git a/cloudsmith_cli/cli/commands/push.py b/cloudsmith_cli/cli/commands/push.py index d16fa3e2..a04ac4d5 100644 --- a/cloudsmith_cli/cli/commands/push.py +++ b/cloudsmith_cli/cli/commands/push.py @@ -38,6 +38,7 @@ validate_create_package as api_validate_create_package, ) from .. import command, decorators, utils, validators +from ..dsc_parser import resolve_dsc_files from ..exceptions import handle_api_exceptions from ..metadata_common import ( MetadataContentError, @@ -1181,6 +1182,22 @@ def push_handler(ctx, *args, **kwargs): parameters = context.get(ctx.info_name) kwargs["package_type"] = ctx.info_name + # deb-only: derive --sources-file/--changes-file from a .dsc's + # Files:/Checksums-Sha256: stanza (see GitHub issue #56). Only + # the `deb` subcommand ever registers --dsc-file (below), so + # this is a no-op kwargs.pop() for every other format. + dsc_file = kwargs.pop("dsc_file", None) + if dsc_file: + dsc_sources_file, dsc_changes_file = resolve_dsc_files(dsc_file) + # Precedence: explicit --sources-file/--changes-file always + # win over values derived from --dsc-file. A user who passes + # both wants a manual override, not to have their explicit + # flag silently clobbered. + if not kwargs.get("sources_file"): + kwargs["sources_file"] = dsc_sources_file + if not kwargs.get("changes_file"): + kwargs["changes_file"] = dsc_changes_file + owner_repo = kwargs.pop("owner_repo") if "distribution" in parameters: kwargs["distribution"] = "/".join(owner_repo[2:]) @@ -1307,6 +1324,32 @@ def push_handler(ctx, *args, **kwargs): ) push_handler = decorator(push_handler) + if key == "deb": + # deb-only shim (GitHub issue #56): the generic per-format loop + # above is driven entirely by the API's package-format model, so + # this stays layered on top rather than becoming a new branch in + # that generic system. + push_handler = click.option( + "--dsc-file", + "dsc_file", + type=click.Path( + exists=True, dir_okay=False, readable=True, resolve_path=True + ), + default=None, + help=( + "Path to a Debian .dsc control file. Its 'Files:' (or " + "'Checksums-Sha256:') stanza is parsed to auto-fill " + "--sources-file/--changes-file, resolving referenced " + "filenames relative to the .dsc's directory. Any " + "--sources-file/--changes-file passed explicitly always " + "takes precedence over a value derived from --dsc-file. " + "Rejected with an error if the .dsc references a " + "multi-component source package (*.orig-*.tar.*) or a " + "detached signature (*.asc) -- Cloudsmith's deb upload " + "format does not support either." + ), + )(push_handler) + handlers[key] = push_handler diff --git a/cloudsmith_cli/cli/dsc_parser.py b/cloudsmith_cli/cli/dsc_parser.py new file mode 100644 index 00000000..e829a076 --- /dev/null +++ b/cloudsmith_cli/cli/dsc_parser.py @@ -0,0 +1,169 @@ +"""CLI - Parse Debian ``.dsc`` control files for ``push deb --dsc-file``. + +A ``.dsc`` (Debian source control) file is an RFC822-style control file that +lists the other files making up a Debian source package (the orig tarball, +the debian packaging tarball/diff, and optionally a detached signature or +extra "component" tarballs) in its ``Files:`` and/or ``Checksums-Sha256:`` +stanza. This module extracts those filenames so ``cloudsmith push deb`` can +derive ``--sources-file``/``--changes-file`` automatically instead of +requiring both to be passed by hand (see GitHub issue #56). + +Only a single-tarball, non-signed source package is supported, matching what +Cloudsmith's ``deb`` package-upload format actually accepts: + +- Multi-component source packages (extra ``*.orig-component.tar.*`` files) + and detached upstream signatures (``*.asc``) referenced by the ``.dsc`` + are rejected with a clear error rather than silently dropped, since the + Cloudsmith backend has nowhere to put them. +""" + +import os +from email.parser import Parser + +import click + +#: Marker identifying a multi-component source tarball reference +#: (e.g. ``foo_1.0.orig-libbar.tar.gz``). Cloudsmith's ``deb`` upload format +#: only accepts a single source tarball, so a ``.dsc`` referencing one of +#: these can't be represented and must be rejected rather than guessed at. +_MULTI_COMPONENT_MARKER = ".orig-" + +#: Suffix identifying a detached signature file (e.g. a ``.dsc.asc`` or an +#: ``.orig.tar.gz.asc``). Cloudsmith's ``deb`` upload format has no field for +#: a detached signature, so these must be rejected rather than silently +#: dropped. +_DETACHED_SIGNATURE_SUFFIX = ".asc" + +_CHANGES_SUFFIX = ".changes" +_DSC_SUFFIX = ".dsc" + +#: The stanza names that carry the file listing in a ``.dsc``, in the order +#: they're checked. ``Files:`` (MD5) is present in every ``.dsc``; +#: ``Checksums-Sha256:`` is the modern equivalent. Either is sufficient. +_FILE_LIST_FIELDS = ("Files", "Checksums-Sha256") + + +def _read_control_message(dsc_path): + """Parse ``dsc_path`` as an RFC822-style control file. + + Uses the stdlib ``email.parser`` rather than a new dependency (e.g. + ``python-debian``) since a ``.dsc``'s single stanza is plain RFC822 + headers followed by (in modern ``.dsc`` files) an inline PGP signature, + which we don't need to verify or even skip explicitly -- the signature + lines simply aren't valid headers and are ignored by the permissive + parser. + """ + try: + with open(dsc_path, encoding="utf-8", errors="replace") as dsc_fh: + return Parser().parse(dsc_fh) + except OSError as exc: + raise click.UsageError( + f"Could not read --dsc-file {dsc_path!r}: {exc}" + ) from exc + + +def _extract_filenames(message, dsc_path): + """Return the filenames listed in a ``.dsc``'s file-listing stanza. + + Each non-blank line of ``Files:``/``Checksums-Sha256:`` looks like + `` [
] ``; only the + trailing filename token is needed. + """ + for field_name in _FILE_LIST_FIELDS: + field_value = message.get(field_name) + if not field_value: + continue + + filenames = [ + line.split()[-1] for line in field_value.splitlines() if line.split() + ] + if filenames: + return filenames + + raise click.UsageError( + f"--dsc-file {dsc_path!r} has no (non-empty) 'Files:' or " + "'Checksums-Sha256:' stanza to parse." + ) + + +def resolve_dsc_files(dsc_path): + """Resolve the source tarball and (optional) changes file for a ``.dsc``. + + Returns a ``(sources_file, changes_file)`` tuple of paths resolved + relative to the directory containing ``dsc_path``. ``changes_file`` is + ``None`` when the ``.dsc`` doesn't reference one -- a ``.dsc`` describes + the source package itself, and pairing it with a ``.changes`` file is + optional (e.g. when the package was never built/uploaded with + ``dpkg-genchanges``). + + Raises ``click.UsageError`` when: + + - the ``.dsc`` references a detached signature (``*.asc``) or a + multi-component source tarball (``*.orig-.tar.*``) -- + Cloudsmith's ``deb`` upload format doesn't support either, so this + fails loudly instead of silently dropping the reference or uploading + the wrong file. + - the remaining files don't resolve to exactly one source tarball, or + more than one ``.changes`` file. + - a referenced file doesn't exist on disk next to the ``.dsc``. + """ + message = _read_control_message(dsc_path) + filenames = _extract_filenames(message, dsc_path) + + signature_files = [f for f in filenames if f.endswith(_DETACHED_SIGNATURE_SUFFIX)] + if signature_files: + raise click.UsageError( + "--dsc-file {dsc!r} references a detached signature file ({files}). " + "Cloudsmith does not support detached upstream signatures for deb " + "uploads.".format(dsc=dsc_path, files=", ".join(signature_files)) + ) + + multi_component_files = [f for f in filenames if _MULTI_COMPONENT_MARKER in f] + if multi_component_files: + raise click.UsageError( + "--dsc-file {dsc!r} references a multi-component source package " + "({files}). Cloudsmith does not support multi-component Debian " + "source packages for deb uploads.".format( + dsc=dsc_path, files=", ".join(multi_component_files) + ) + ) + + changes_files = [f for f in filenames if f.endswith(_CHANGES_SUFFIX)] + if len(changes_files) > 1: + raise click.UsageError( + "--dsc-file {dsc!r} references more than one .changes file " + "({files}); expected at most one.".format( + dsc=dsc_path, files=", ".join(changes_files) + ) + ) + + source_files = [ + f for f in filenames if f not in changes_files and not f.endswith(_DSC_SUFFIX) + ] + if not source_files: + raise click.UsageError( + f"--dsc-file {dsc_path!r} does not reference a source tarball." + ) + if len(source_files) > 1: + raise click.UsageError( + "--dsc-file {dsc!r} references more than one source tarball " + "({files}); expected exactly one non-signature, " + "non-multi-component file.".format( + dsc=dsc_path, files=", ".join(source_files) + ) + ) + + base_dir = os.path.dirname(os.path.abspath(dsc_path)) + + def _resolve(filename): + resolved = os.path.join(base_dir, filename) + if not os.path.isfile(resolved): + raise click.UsageError( + f"--dsc-file {dsc_path!r} references {filename!r}, but it " + f"was not found next to the .dsc (expected at {resolved!r})." + ) + return resolved + + sources_file = _resolve(source_files[0]) + changes_file = _resolve(changes_files[0]) if changes_files else None + return sources_file, changes_file diff --git a/cloudsmith_cli/cli/tests/test_dsc_parser.py b/cloudsmith_cli/cli/tests/test_dsc_parser.py new file mode 100644 index 00000000..29f1685f --- /dev/null +++ b/cloudsmith_cli/cli/tests/test_dsc_parser.py @@ -0,0 +1,122 @@ +"""Tests for ``cloudsmith_cli.cli.dsc_parser`` (GitHub issue #56).""" + +import click +import pytest + +from ..dsc_parser import resolve_dsc_files + + +def _write_dsc(tmp_path, file_lines, field="Files", name="pkg_1.0-1.dsc"): + """Write a minimal .dsc control file listing ``file_lines`` under ``field``.""" + body_lines = "\n".join(f" deadbeef 100 {line}" for line in file_lines) + dsc_path = tmp_path / name + dsc_path.write_text( + f"Format: 3.0 (native)\nSource: pkg\nVersion: 1.0-1\n{field}:\n{body_lines}\n" + ) + return dsc_path + + +def _touch(tmp_path, name): + path = tmp_path / name + path.write_bytes(b"dummy content") + return path + + +def test_resolve_dsc_files_happy_path_with_changes(tmp_path): + _touch(tmp_path, "pkg_1.0.tar.gz") + _touch(tmp_path, "pkg_1.0-1_amd64.changes") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz", "pkg_1.0-1_amd64.changes"]) + + sources_file, changes_file = resolve_dsc_files(str(dsc_path)) + + assert sources_file == str(tmp_path / "pkg_1.0.tar.gz") + assert changes_file == str(tmp_path / "pkg_1.0-1_amd64.changes") + + +def test_resolve_dsc_files_happy_path_without_changes(tmp_path): + _touch(tmp_path, "pkg_1.0.tar.gz") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"]) + + sources_file, changes_file = resolve_dsc_files(str(dsc_path)) + + assert sources_file == str(tmp_path / "pkg_1.0.tar.gz") + assert changes_file is None + + +def test_resolve_dsc_files_uses_checksums_sha256_fallback(tmp_path): + _touch(tmp_path, "pkg_1.0.tar.gz") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"], field="Checksums-Sha256") + + sources_file, _ = resolve_dsc_files(str(dsc_path)) + + assert sources_file == str(tmp_path / "pkg_1.0.tar.gz") + + +def test_resolve_dsc_files_missing_referenced_file_errors(tmp_path): + # pkg_1.0.tar.gz is referenced but never actually written to disk. + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"]) + + with pytest.raises(click.UsageError, match="not found next to the .dsc"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolve_dsc_files_rejects_multi_component(tmp_path): + _touch(tmp_path, "pkg_1.0.orig.tar.gz") + _touch(tmp_path, "pkg_1.0.orig-libbar.tar.gz") + dsc_path = _write_dsc( + tmp_path, ["pkg_1.0.orig.tar.gz", "pkg_1.0.orig-libbar.tar.gz"] + ) + + with pytest.raises(click.UsageError, match="multi-component"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolve_dsc_files_rejects_detached_signature(tmp_path): + _touch(tmp_path, "pkg_1.0.tar.gz") + _touch(tmp_path, "pkg_1.0.tar.gz.asc") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz", "pkg_1.0.tar.gz.asc"]) + + with pytest.raises(click.UsageError, match="detached signature"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolve_dsc_files_rejects_ambiguous_multiple_source_tarballs(tmp_path): + _touch(tmp_path, "pkg_1.0.tar.gz") + _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz", "pkg_1.0-1.debian.tar.xz"]) + + with pytest.raises(click.UsageError, match="more than one source tarball"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolve_dsc_files_rejects_multiple_changes_files(tmp_path): + _touch(tmp_path, "pkg_1.0.tar.gz") + _touch(tmp_path, "pkg_1.0-1_amd64.changes") + _touch(tmp_path, "pkg_1.0-1_source.changes") + dsc_path = _write_dsc( + tmp_path, + [ + "pkg_1.0.tar.gz", + "pkg_1.0-1_amd64.changes", + "pkg_1.0-1_source.changes", + ], + ) + + with pytest.raises(click.UsageError, match="more than one .changes file"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolve_dsc_files_no_files_stanza_errors(tmp_path): + dsc_path = tmp_path / "empty.dsc" + dsc_path.write_text("Format: 3.0 (native)\nSource: pkg\nVersion: 1.0-1\n") + + with pytest.raises(click.UsageError, match="Files"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolve_dsc_files_no_source_tarball_errors(tmp_path): + _touch(tmp_path, "pkg_1.0-1_amd64.changes") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0-1_amd64.changes"]) + + with pytest.raises(click.UsageError, match="does not reference a source tarball"): + resolve_dsc_files(str(dsc_path)) diff --git a/cloudsmith_cli/cli/tests/test_push_dsc_cli.py b/cloudsmith_cli/cli/tests/test_push_dsc_cli.py new file mode 100644 index 00000000..5f30c0cc --- /dev/null +++ b/cloudsmith_cli/cli/tests/test_push_dsc_cli.py @@ -0,0 +1,181 @@ +"""CLI-level tests for `cloudsmith push deb --dsc-file` (GitHub issue #56). + +Complements ``test_dsc_parser.py`` (pure ``.dsc`` parsing/resolution) with +integration coverage of the ``deb`` push handler: the flag itself, the +--sources-file/--changes-file precedence rule, and that a rejected .dsc +aborts before any network call. +""" + +from unittest.mock import patch + +import pytest + +from .. import config as cli_config +from ..commands.push import push + +HERMETIC_ARGS = ["--api-key", "fake-api-key"] + +_MOCK_TARGETS = ( + "cloudsmith_cli.cli.commands.push.validate_create_package", + "cloudsmith_cli.cli.commands.push.validate_upload_file", + "cloudsmith_cli.cli.commands.push.upload_file", + "cloudsmith_cli.cli.commands.push.create_package", + "cloudsmith_cli.cli.commands.push.wait_for_package_sync", +) + + +@pytest.fixture(autouse=True) +def hermetic_environment(monkeypatch): + """Keep a developer's real env/config out of these tests (see test_domains.py).""" + monkeypatch.delenv("CLOUDSMITH_ORG", raising=False) + monkeypatch.delenv("CLOUDSMITH_CONFIG_FILE", raising=False) + monkeypatch.delattr(cli_config.OPTIONS, "value", raising=False) + monkeypatch.setattr(cli_config.ConfigReader, "config_files", ["config.ini"]) + monkeypatch.setattr(cli_config.ConfigReader, "config_searchpath", ["."]) + + +def _touch(tmp_path, name): + path = tmp_path / name + path.write_bytes(b"dummy content") + return path + + +def _write_dsc(tmp_path, file_lines, field="Files", name="pkg_1.0-1.dsc"): + body = "\n".join(f" deadbeef 100 {line}" for line in file_lines) + dsc_path = tmp_path / name + dsc_path.write_text( + f"Format: 3.0 (native)\nSource: pkg\nVersion: 1.0-1\n{field}:\n{body}\n" + ) + return dsc_path + + +def _invoke(runner, tmp_path, extra_args, mocks): + package_file = _touch(tmp_path, "pkg_1.0-1_amd64.deb") + with ( + patch(_MOCK_TARGETS[0]) as mock_validate_create_package, + patch(_MOCK_TARGETS[1], return_value="checksum"), + patch(_MOCK_TARGETS[2], return_value="file-id"), + patch(_MOCK_TARGETS[3], return_value=("slug-perm", "slug")), + patch(_MOCK_TARGETS[4]), + ): + mocks["validate_create_package"] = mock_validate_create_package + result = runner.invoke( + push, + [ + "deb", + "acme/repo/ubuntu/xenial", + str(package_file), + *extra_args, + *HERMETIC_ARGS, + ], + catch_exceptions=False, + ) + return result + + +def test_dsc_file_derives_sources_and_changes_files(runner, tmp_path): + sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") + changes_file = _touch(tmp_path, "pkg_1.0-1_amd64.changes") + dsc_path = _write_dsc(tmp_path, [sources_tarball.name, changes_file.name]) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + + assert result.exit_code == 0, result.output + kwargs = mocks["validate_create_package"].call_args.kwargs + assert kwargs["sources_file"] == str(sources_tarball) + assert kwargs["changes_file"] == str(changes_file) + + +def test_explicit_sources_file_wins_over_dsc_file(runner, tmp_path): + """Explicit --sources-file always takes precedence over --dsc-file.""" + dsc_sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") + changes_file = _touch(tmp_path, "pkg_1.0-1_amd64.changes") + explicit_sources_file = _touch(tmp_path, "explicit_sources.tar.gz") + dsc_path = _write_dsc(tmp_path, [dsc_sources_tarball.name, changes_file.name]) + + mocks = {} + result = _invoke( + runner, + tmp_path, + [ + "--dsc-file", + str(dsc_path), + "--sources-file", + str(explicit_sources_file), + ], + mocks, + ) + + assert result.exit_code == 0, result.output + kwargs = mocks["validate_create_package"].call_args.kwargs + # Explicit flag wins... + assert kwargs["sources_file"] == str(explicit_sources_file) + # ...but changes_file, which was not passed explicitly, is still derived. + assert kwargs["changes_file"] == str(changes_file) + + +def test_explicit_changes_file_wins_over_dsc_file(runner, tmp_path): + """Explicit --changes-file always takes precedence over --dsc-file.""" + sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") + dsc_changes_file = _touch(tmp_path, "pkg_1.0-1_amd64.changes") + explicit_changes_file = _touch(tmp_path, "explicit.changes") + dsc_path = _write_dsc(tmp_path, [sources_tarball.name, dsc_changes_file.name]) + + mocks = {} + result = _invoke( + runner, + tmp_path, + [ + "--dsc-file", + str(dsc_path), + "--changes-file", + str(explicit_changes_file), + ], + mocks, + ) + + assert result.exit_code == 0, result.output + kwargs = mocks["validate_create_package"].call_args.kwargs + assert kwargs["sources_file"] == str(sources_tarball) + assert kwargs["changes_file"] == str(explicit_changes_file) + + +def test_dsc_file_rejects_multi_component_before_any_network_call(runner, tmp_path): + orig_tarball = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + component_tarball = _touch(tmp_path, "pkg_1.0.orig-libbar.tar.gz") + dsc_path = _write_dsc(tmp_path, [orig_tarball.name, component_tarball.name]) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + + assert result.exit_code != 0 + assert "multi-component" in result.output + mocks["validate_create_package"].assert_not_called() + + +def test_dsc_file_rejects_detached_signature_before_any_network_call(runner, tmp_path): + sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") + signature_file = _touch(tmp_path, "pkg_1.0.tar.gz.asc") + dsc_path = _write_dsc(tmp_path, [sources_tarball.name, signature_file.name]) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + + assert result.exit_code != 0 + assert "detached signature" in result.output + mocks["validate_create_package"].assert_not_called() + + +def test_push_deb_help_documents_dsc_file_option(runner): + result = runner.invoke(push, ["deb", "--help"], catch_exceptions=False) + + assert result.exit_code == 0, result.output + assert "--dsc-file" in result.output + + +def test_push_non_deb_format_has_no_dsc_file_option(runner): + result = runner.invoke(push, ["raw", "--help"], catch_exceptions=False) + + assert result.exit_code == 0, result.output + assert "--dsc-file" not in result.output From c5c7d2667b9c60ebf8465a76d8d14f15f152becd Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Thu, 13 Aug 2026 09:26:41 +0100 Subject: [PATCH 2/4] fix(ENG-56): safely parse Debian source package members --- cloudsmith_cli/cli/commands/push.py | 16 +- cloudsmith_cli/cli/dsc_parser.py | 337 +++++++++++------- cloudsmith_cli/cli/tests/test_dsc_parser.py | 248 +++++++++---- cloudsmith_cli/cli/tests/test_push_dsc_cli.py | 238 ++++++++++--- 4 files changed, 584 insertions(+), 255 deletions(-) diff --git a/cloudsmith_cli/cli/commands/push.py b/cloudsmith_cli/cli/commands/push.py index a04ac4d5..786c8cf0 100644 --- a/cloudsmith_cli/cli/commands/push.py +++ b/cloudsmith_cli/cli/commands/push.py @@ -1182,10 +1182,10 @@ def push_handler(ctx, *args, **kwargs): parameters = context.get(ctx.info_name) kwargs["package_type"] = ctx.info_name - # deb-only: derive --sources-file/--changes-file from a .dsc's - # Files:/Checksums-Sha256: stanza (see GitHub issue #56). Only - # the `deb` subcommand ever registers --dsc-file (below), so - # this is a no-op kwargs.pop() for every other format. + # deb-only: derive the upstream/native source archive and Debian + # packaging archive from a .dsc (see GitHub issue #56). Only the + # `deb` subcommand ever registers --dsc-file (below), so this is + # a no-op kwargs.pop() for every other format. dsc_file = kwargs.pop("dsc_file", None) if dsc_file: dsc_sources_file, dsc_changes_file = resolve_dsc_files(dsc_file) @@ -1338,9 +1338,11 @@ def push_handler(ctx, *args, **kwargs): default=None, help=( "Path to a Debian .dsc control file. Its 'Files:' (or " - "'Checksums-Sha256:') stanza is parsed to auto-fill " - "--sources-file/--changes-file, resolving referenced " - "filenames relative to the .dsc's directory. Any " + "'Checksums-Sha256:') field is parsed to auto-fill " + "--sources-file with the upstream/native source archive " + "and --changes-file with the Debian packaging archive " + "(.debian.tar.* or .diff.gz). Referenced files must be " + "directly next to the .dsc. Any " "--sources-file/--changes-file passed explicitly always " "takes precedence over a value derived from --dsc-file. " "Rejected with an error if the .dsc references a " diff --git a/cloudsmith_cli/cli/dsc_parser.py b/cloudsmith_cli/cli/dsc_parser.py index e829a076..1f6d1ab8 100644 --- a/cloudsmith_cli/cli/dsc_parser.py +++ b/cloudsmith_cli/cli/dsc_parser.py @@ -1,169 +1,244 @@ -"""CLI - Parse Debian ``.dsc`` control files for ``push deb --dsc-file``. - -A ``.dsc`` (Debian source control) file is an RFC822-style control file that -lists the other files making up a Debian source package (the orig tarball, -the debian packaging tarball/diff, and optionally a detached signature or -extra "component" tarballs) in its ``Files:`` and/or ``Checksums-Sha256:`` -stanza. This module extracts those filenames so ``cloudsmith push deb`` can -derive ``--sources-file``/``--changes-file`` automatically instead of -requiring both to be passed by hand (see GitHub issue #56). - -Only a single-tarball, non-signed source package is supported, matching what -Cloudsmith's ``deb`` package-upload format actually accepts: - -- Multi-component source packages (extra ``*.orig-component.tar.*`` files) - and detached upstream signatures (``*.asc``) referenced by the ``.dsc`` - are rejected with a clear error rather than silently dropped, since the - Cloudsmith backend has nowhere to put them. -""" +"""Parse Debian ``.dsc`` control files for ``push deb --dsc-file``.""" import os from email.parser import Parser import click -#: Marker identifying a multi-component source tarball reference -#: (e.g. ``foo_1.0.orig-libbar.tar.gz``). Cloudsmith's ``deb`` upload format -#: only accepts a single source tarball, so a ``.dsc`` referencing one of -#: these can't be represented and must be rejected rather than guessed at. -_MULTI_COMPONENT_MARKER = ".orig-" - -#: Suffix identifying a detached signature file (e.g. a ``.dsc.asc`` or an -#: ``.orig.tar.gz.asc``). Cloudsmith's ``deb`` upload format has no field for -#: a detached signature, so these must be rejected rather than silently -#: dropped. +_CLEAR_SIGNED_MESSAGE = "-----BEGIN PGP SIGNED MESSAGE-----" +_SIGNATURE = "-----BEGIN PGP SIGNATURE-----" _DETACHED_SIGNATURE_SUFFIX = ".asc" +_FILE_LIST_FIELDS = ("Checksums-Sha256", "Files") +_QUILT_FORMATS = {"2.0", "3.0 (quilt)"} +_SINGLE_TARBALL_FORMATS = {"3.0 (native)", "3.0 (git)", "3.0 (bzr)"} + + +def _usage_error(dsc_path, message): + return click.UsageError(f"--dsc-file {dsc_path!r} {message}") -_CHANGES_SUFFIX = ".changes" -_DSC_SUFFIX = ".dsc" -#: The stanza names that carry the file listing in a ``.dsc``, in the order -#: they're checked. ``Files:`` (MD5) is present in every ``.dsc``; -#: ``Checksums-Sha256:`` is the modern equivalent. Either is sufficient. -_FILE_LIST_FIELDS = ("Files", "Checksums-Sha256") +def _unwrap_clearsigned_control(text, dsc_path): + """Return deb822 control text from an optional OpenPGP cleartext signature.""" + lines = text.splitlines(keepends=True) + if not lines or lines[0].rstrip("\r\n") != _CLEAR_SIGNED_MESSAGE: + return text + + index = 1 + while index < len(lines) and lines[index].rstrip("\r\n"): + index += 1 + if index == len(lines): + raise _usage_error(dsc_path, "has a malformed OpenPGP cleartext signature.") + + cleartext = [] + for line in lines[index + 1 :]: + if line.rstrip("\r\n") == _SIGNATURE: + return "".join(cleartext) + # RFC 9580 cleartext signatures dash-escape lines beginning with "-". + cleartext.append(line.removeprefix("- ")) + + raise _usage_error(dsc_path, "has a malformed OpenPGP cleartext signature.") def _read_control_message(dsc_path): - """Parse ``dsc_path`` as an RFC822-style control file. - - Uses the stdlib ``email.parser`` rather than a new dependency (e.g. - ``python-debian``) since a ``.dsc``'s single stanza is plain RFC822 - headers followed by (in modern ``.dsc`` files) an inline PGP signature, - which we don't need to verify or even skip explicitly -- the signature - lines simply aren't valid headers and are ignored by the permissive - parser. - """ + """Parse ``dsc_path`` as a plain or OpenPGP-clearsigned deb822 stanza.""" try: with open(dsc_path, encoding="utf-8", errors="replace") as dsc_fh: - return Parser().parse(dsc_fh) + text = dsc_fh.read() except OSError as exc: - raise click.UsageError( - f"Could not read --dsc-file {dsc_path!r}: {exc}" - ) from exc + raise _usage_error(dsc_path, f"could not be read: {exc}") from exc + return Parser().parsestr(_unwrap_clearsigned_control(text, dsc_path)) -def _extract_filenames(message, dsc_path): - """Return the filenames listed in a ``.dsc``'s file-listing stanza. - Each non-blank line of ``Files:``/``Checksums-Sha256:`` looks like - `` [
] ``; only the - trailing filename token is needed. - """ +def _parse_file_list(field_name, field_value, dsc_path): + filenames = [] + for line in field_value.splitlines(): + if not line.strip(): + continue + parts = line.split() + if len(parts) != 3: + raise _usage_error( + dsc_path, + f"has a malformed {field_name!r} entry {line.strip()!r}; " + "expected ' '.", + ) + filenames.append(parts[2]) + return filenames + + +def _extract_filenames(message, dsc_path): + """Return an agreed filename list, preferring the strong SHA-256 field.""" + file_lists = {} for field_name in _FILE_LIST_FIELDS: field_value = message.get(field_name) - if not field_value: - continue + if field_value: + filenames = _parse_file_list(field_name, field_value, dsc_path) + if filenames: + file_lists[field_name] = filenames + + if not file_lists: + raise _usage_error( + dsc_path, + "has no non-empty 'Checksums-Sha256:' or 'Files:' field to parse.", + ) - filenames = [ - line.split()[-1] for line in field_value.splitlines() if line.split() - ] - if filenames: - return filenames + for filenames in file_lists.values(): + if len(filenames) != len(set(filenames)): + raise _usage_error(dsc_path, "lists the same source-package member twice.") + + selected_name = next(iter(file_lists)) + selected = file_lists[selected_name] + for field_name, filenames in file_lists.items(): + if set(filenames) != set(selected): + raise _usage_error( + dsc_path, + f"has conflicting filenames in {selected_name!r} and {field_name!r}.", + ) - raise click.UsageError( - f"--dsc-file {dsc_path!r} has no (non-empty) 'Files:' or " - "'Checksums-Sha256:' stanza to parse." - ) + return selected -def resolve_dsc_files(dsc_path): - """Resolve the source tarball and (optional) changes file for a ``.dsc``. - - Returns a ``(sources_file, changes_file)`` tuple of paths resolved - relative to the directory containing ``dsc_path``. ``changes_file`` is - ``None`` when the ``.dsc`` doesn't reference one -- a ``.dsc`` describes - the source package itself, and pairing it with a ``.changes`` file is - optional (e.g. when the package was never built/uploaded with - ``dpkg-genchanges``). - - Raises ``click.UsageError`` when: - - - the ``.dsc`` references a detached signature (``*.asc``) or a - multi-component source tarball (``*.orig-.tar.*``) -- - Cloudsmith's ``deb`` upload format doesn't support either, so this - fails loudly instead of silently dropping the reference or uploading - the wrong file. - - the remaining files don't resolve to exactly one source tarball, or - more than one ``.changes`` file. - - a referenced file doesn't exist on disk next to the ``.dsc``. - """ - message = _read_control_message(dsc_path) - filenames = _extract_filenames(message, dsc_path) +def _required_field(message, field_name, dsc_path): + value = message.get(field_name) + if not value or not value.strip(): + raise _usage_error(dsc_path, f"has no {field_name!r} field.") + return value.strip() - signature_files = [f for f in filenames if f.endswith(_DETACHED_SIGNATURE_SUFFIX)] + +def _validate_member_names(filenames, dsc_path): + for filename in filenames: + if os.path.isabs(filename) or os.path.basename(filename) != filename: + raise _usage_error( + dsc_path, + f"references invalid member filename {filename!r}; source-package " + "members must be filenames next to the .dsc, not paths.", + ) + + +def _is_tar_archive(filename, stem): + prefix = f"{stem}.tar." + return filename.startswith(prefix) and len(filename) > len(prefix) + + +def _classify_members(message, filenames, dsc_path): + """Map source-package members to the two fields accepted by the SDK.""" + source = _required_field(message, "Source", dsc_path) + version = _required_field(message, "Version", dsc_path).split(":", 1)[-1] + source_format = _required_field(message, "Format", dsc_path) + upstream_version = version.rsplit("-", 1)[0] + + signature_files = [ + filename + for filename in filenames + if filename.endswith(_DETACHED_SIGNATURE_SUFFIX) + ] if signature_files: - raise click.UsageError( - "--dsc-file {dsc!r} references a detached signature file ({files}). " - "Cloudsmith does not support detached upstream signatures for deb " - "uploads.".format(dsc=dsc_path, files=", ".join(signature_files)) + raise _usage_error( + dsc_path, + "references detached upstream signature file(s) ({files}). Cloudsmith " + "does not support detached upstream signatures for deb uploads.".format( + files=", ".join(signature_files) + ), ) - multi_component_files = [f for f in filenames if _MULTI_COMPONENT_MARKER in f] - if multi_component_files: - raise click.UsageError( - "--dsc-file {dsc!r} references a multi-component source package " - "({files}). Cloudsmith does not support multi-component Debian " - "source packages for deb uploads.".format( - dsc=dsc_path, files=", ".join(multi_component_files) - ) + sources = [] + changes = [] + component_files = [] + legacy_non_native = False + + if source_format in _QUILT_FORMATS: + orig_stem = f"{source}_{upstream_version}.orig" + component_prefix = f"{orig_stem}-" + debian_stem = f"{source}_{version}.debian" + sources = [f for f in filenames if _is_tar_archive(f, orig_stem)] + changes = [f for f in filenames if _is_tar_archive(f, debian_stem)] + component_files = [ + f + for f in filenames + if f.startswith(component_prefix) and ".tar." in f[len(component_prefix) :] + ] + elif source_format == "1.0": + orig_stem = f"{source}_{upstream_version}.orig" + diff_name = f"{source}_{version}.diff.gz" + native_stem = f"{source}_{version}" + orig_files = [f for f in filenames if _is_tar_archive(f, orig_stem)] + diff_files = [f for f in filenames if f == diff_name] + native_files = [f for f in filenames if _is_tar_archive(f, native_stem)] + if orig_files or diff_files: + legacy_non_native = True + sources = orig_files + changes = diff_files + else: + sources = native_files + elif source_format in _SINGLE_TARBALL_FORMATS: + sources = [f for f in filenames if _is_tar_archive(f, f"{source}_{version}")] + else: + raise _usage_error( + dsc_path, f"uses unsupported Debian source format {source_format!r}." ) - changes_files = [f for f in filenames if f.endswith(_CHANGES_SUFFIX)] - if len(changes_files) > 1: - raise click.UsageError( - "--dsc-file {dsc!r} references more than one .changes file " - "({files}); expected at most one.".format( - dsc=dsc_path, files=", ".join(changes_files) - ) + if component_files: + raise _usage_error( + dsc_path, + "references a multi-component source package ({files}). Cloudsmith " + "does not support multi-component Debian source packages for deb " + "uploads.".format(files=", ".join(component_files)), ) - source_files = [ - f for f in filenames if f not in changes_files and not f.endswith(_DSC_SUFFIX) - ] - if not source_files: - raise click.UsageError( - f"--dsc-file {dsc_path!r} does not reference a source tarball." + classified = set(sources + changes + signature_files + component_files) + unexpected = [f for f in filenames if f not in classified] + if unexpected: + raise _usage_error( + dsc_path, + "contains unsupported or incorrectly named source-package member(s) " + f"({', '.join(unexpected)}) for format {source_format!r}.", ) - if len(source_files) > 1: - raise click.UsageError( - "--dsc-file {dsc!r} references more than one source tarball " - "({files}); expected exactly one non-signature, " - "non-multi-component file.".format( - dsc=dsc_path, files=", ".join(source_files) - ) + if len(sources) != 1: + raise _usage_error( + dsc_path, + f"must reference exactly one main source archive for format " + f"{source_format!r}; found {len(sources)}.", + ) + if source_format in _QUILT_FORMATS and len(changes) != 1: + raise _usage_error( + dsc_path, + f"must reference exactly one Debian packaging archive for format " + f"{source_format!r}; found {len(changes)}.", + ) + if legacy_non_native and len(changes) != 1: + raise _usage_error( + dsc_path, + "must reference exactly one Debian packaging diff for non-native " + f"format '1.0'; found {len(changes)}.", ) - base_dir = os.path.dirname(os.path.abspath(dsc_path)) + return sources[0], changes[0] if changes else None - def _resolve(filename): - resolved = os.path.join(base_dir, filename) - if not os.path.isfile(resolved): - raise click.UsageError( - f"--dsc-file {dsc_path!r} references {filename!r}, but it " - f"was not found next to the .dsc (expected at {resolved!r})." - ) - return resolved - sources_file = _resolve(source_files[0]) - changes_file = _resolve(changes_files[0]) if changes_files else None +def _resolve_member(base_dir, filename, dsc_path): + candidate = os.path.join(base_dir, filename) + resolved = os.path.realpath(candidate) + if os.path.dirname(resolved) != base_dir or not os.path.isfile(resolved): + raise _usage_error( + dsc_path, + f"references {filename!r}, but it is not a regular file directly " + "next to the .dsc.", + ) + return resolved + + +def resolve_dsc_files(dsc_path): + """Return ``(sources_file, changes_file)`` derived from a Debian ``.dsc``.""" + message = _read_control_message(dsc_path) + filenames = _extract_filenames(message, dsc_path) + _validate_member_names(filenames, dsc_path) + source_filename, changes_filename = _classify_members(message, filenames, dsc_path) + + base_dir = os.path.realpath(os.path.dirname(os.path.abspath(dsc_path))) + sources_file = _resolve_member(base_dir, source_filename, dsc_path) + changes_file = ( + _resolve_member(base_dir, changes_filename, dsc_path) + if changes_filename + else None + ) return sources_file, changes_file diff --git a/cloudsmith_cli/cli/tests/test_dsc_parser.py b/cloudsmith_cli/cli/tests/test_dsc_parser.py index 29f1685f..6b28d2a9 100644 --- a/cloudsmith_cli/cli/tests/test_dsc_parser.py +++ b/cloudsmith_cli/cli/tests/test_dsc_parser.py @@ -1,4 +1,6 @@ -"""Tests for ``cloudsmith_cli.cli.dsc_parser`` (GitHub issue #56).""" +"""Tests for Debian ``.dsc`` parsing and member resolution.""" + +import os import click import pytest @@ -6,13 +8,38 @@ from ..dsc_parser import resolve_dsc_files -def _write_dsc(tmp_path, file_lines, field="Files", name="pkg_1.0-1.dsc"): - """Write a minimal .dsc control file listing ``file_lines`` under ``field``.""" - body_lines = "\n".join(f" deadbeef 100 {line}" for line in file_lines) - dsc_path = tmp_path / name - dsc_path.write_text( - f"Format: 3.0 (native)\nSource: pkg\nVersion: 1.0-1\n{field}:\n{body_lines}\n" +def _file_field(name, filenames): + checksum = "a" * (64 if name == "Checksums-Sha256" else 32) + entries = "\n".join(f" {checksum} 100 {filename}" for filename in filenames) + return f"{name}:\n{entries}\n" + + +def _write_dsc( + tmp_path, + filenames, + *, + source="pkg", + version="1.0", + source_format="3.0 (native)", + fields=("Files",), + clearsigned=False, + name=None, +): + control = ( + f"Format: {source_format}\nSource: {source}\nVersion: {version}\n" + + "".join(_file_field(field, filenames) for field in fields) ) + if clearsigned: + control = ( + "-----BEGIN PGP SIGNED MESSAGE-----\n" + "Hash: SHA256\n\n" + f"{control}" + "-----BEGIN PGP SIGNATURE-----\n" + "test-signature-data\n" + "-----END PGP SIGNATURE-----\n" + ) + dsc_path = tmp_path / (name or f"{source}_{version}.dsc") + dsc_path.write_text(control) return dsc_path @@ -22,101 +49,202 @@ def _touch(tmp_path, name): return path -def test_resolve_dsc_files_happy_path_with_changes(tmp_path): - _touch(tmp_path, "pkg_1.0.tar.gz") - _touch(tmp_path, "pkg_1.0-1_amd64.changes") - dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz", "pkg_1.0-1_amd64.changes"]) +def test_resolves_native_source_archive(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.tar.xz") + dsc_path = _write_dsc(tmp_path, [source_archive.name]) - sources_file, changes_file = resolve_dsc_files(str(dsc_path)) + assert resolve_dsc_files(str(dsc_path)) == (str(source_archive), None) - assert sources_file == str(tmp_path / "pkg_1.0.tar.gz") - assert changes_file == str(tmp_path / "pkg_1.0-1_amd64.changes") +def test_resolves_quilt_source_and_debian_archives(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name, debian_archive.name], + version="1.0-1", + source_format="3.0 (quilt)", + ) + + assert resolve_dsc_files(str(dsc_path)) == ( + str(source_archive), + str(debian_archive), + ) + + +def test_resolves_legacy_non_native_source_and_diff_archives(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + diff_archive = _touch(tmp_path, "pkg_1.0-1.diff.gz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name, diff_archive.name], + version="1.0-1", + source_format="1.0", + ) + + assert resolve_dsc_files(str(dsc_path)) == ( + str(source_archive), + str(diff_archive), + ) -def test_resolve_dsc_files_happy_path_without_changes(tmp_path): - _touch(tmp_path, "pkg_1.0.tar.gz") - dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"]) - sources_file, changes_file = resolve_dsc_files(str(dsc_path)) +def test_rejects_legacy_non_native_source_without_diff(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name], + version="1.0-1", + source_format="1.0", + ) - assert sources_file == str(tmp_path / "pkg_1.0.tar.gz") - assert changes_file is None + with pytest.raises(click.UsageError, match="exactly one Debian packaging diff"): + resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_uses_checksums_sha256_fallback(tmp_path): - _touch(tmp_path, "pkg_1.0.tar.gz") - dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"], field="Checksums-Sha256") +def test_resolves_clearsigned_dsc_using_matching_sha256_and_files(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name, debian_archive.name], + version="1.0-1", + source_format="3.0 (quilt)", + fields=("Files", "Checksums-Sha256"), + clearsigned=True, + ) - sources_file, _ = resolve_dsc_files(str(dsc_path)) + assert resolve_dsc_files(str(dsc_path)) == ( + str(source_archive), + str(debian_archive), + ) - assert sources_file == str(tmp_path / "pkg_1.0.tar.gz") +def test_uses_checksums_sha256_when_files_is_absent(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.tar.gz") + dsc_path = _write_dsc(tmp_path, [source_archive.name], fields=("Checksums-Sha256",)) + + assert resolve_dsc_files(str(dsc_path)) == (str(source_archive), None) -def test_resolve_dsc_files_missing_referenced_file_errors(tmp_path): - # pkg_1.0.tar.gz is referenced but never actually written to disk. - dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"]) - with pytest.raises(click.UsageError, match="not found next to the .dsc"): +def test_rejects_conflicting_checksum_file_lists(tmp_path): + dsc_path = tmp_path / "pkg_1.0.dsc" + dsc_path.write_text( + "Format: 3.0 (native)\nSource: pkg\nVersion: 1.0\n" + + _file_field("Files", ["pkg_1.0.tar.gz"]) + + _file_field("Checksums-Sha256", ["other_1.0.tar.gz"]) + ) + + with pytest.raises(click.UsageError, match="conflicting filenames"): resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_rejects_multi_component(tmp_path): - _touch(tmp_path, "pkg_1.0.orig.tar.gz") - _touch(tmp_path, "pkg_1.0.orig-libbar.tar.gz") - dsc_path = _write_dsc( - tmp_path, ["pkg_1.0.orig.tar.gz", "pkg_1.0.orig-libbar.tar.gz"] +def test_rejects_malformed_file_list_row(tmp_path): + dsc_path = tmp_path / "pkg_1.0.dsc" + dsc_path.write_text( + "Format: 3.0 (native)\nSource: pkg\nVersion: 1.0\n" + "Files:\n deadbeef pkg_1.0.tar.gz\n" ) - with pytest.raises(click.UsageError, match="multi-component"): + with pytest.raises(click.UsageError, match="malformed 'Files' entry"): resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_rejects_detached_signature(tmp_path): - _touch(tmp_path, "pkg_1.0.tar.gz") - _touch(tmp_path, "pkg_1.0.tar.gz.asc") - dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz", "pkg_1.0.tar.gz.asc"]) +def test_rejects_missing_referenced_file(tmp_path): + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"]) - with pytest.raises(click.UsageError, match="detached signature"): + with pytest.raises(click.UsageError, match="not a regular file directly"): resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_rejects_ambiguous_multiple_source_tarballs(tmp_path): - _touch(tmp_path, "pkg_1.0.tar.gz") +def test_rejects_multi_component_source_package(tmp_path): + _touch(tmp_path, "pkg_1.0.orig.tar.gz") + _touch(tmp_path, "pkg_1.0.orig-libbar.tar.gz") _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") - dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz", "pkg_1.0-1.debian.tar.xz"]) + dsc_path = _write_dsc( + tmp_path, + [ + "pkg_1.0.orig.tar.gz", + "pkg_1.0.orig-libbar.tar.gz", + "pkg_1.0-1.debian.tar.xz", + ], + version="1.0-1", + source_format="3.0 (quilt)", + ) - with pytest.raises(click.UsageError, match="more than one source tarball"): + with pytest.raises(click.UsageError, match="multi-component"): resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_rejects_multiple_changes_files(tmp_path): - _touch(tmp_path, "pkg_1.0.tar.gz") - _touch(tmp_path, "pkg_1.0-1_amd64.changes") - _touch(tmp_path, "pkg_1.0-1_source.changes") +def test_rejects_detached_signature(tmp_path): + _touch(tmp_path, "pkg_1.0.orig.tar.gz") + _touch(tmp_path, "pkg_1.0.orig.tar.gz.asc") + _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") dsc_path = _write_dsc( tmp_path, [ - "pkg_1.0.tar.gz", - "pkg_1.0-1_amd64.changes", - "pkg_1.0-1_source.changes", + "pkg_1.0.orig.tar.gz", + "pkg_1.0.orig.tar.gz.asc", + "pkg_1.0-1.debian.tar.xz", ], + version="1.0-1", + source_format="3.0 (quilt)", ) - with pytest.raises(click.UsageError, match="more than one .changes file"): + with pytest.raises(click.UsageError, match="detached upstream signature"): resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_no_files_stanza_errors(tmp_path): - dsc_path = tmp_path / "empty.dsc" - dsc_path.write_text("Format: 3.0 (native)\nSource: pkg\nVersion: 1.0-1\n") +@pytest.mark.parametrize("filename", ["/etc/passwd", "../pkg_1.0.tar.gz"]) +def test_rejects_member_paths(filename, tmp_path): + dsc_path = _write_dsc(tmp_path, [filename]) - with pytest.raises(click.UsageError, match="Files"): + with pytest.raises(click.UsageError, match="must be filenames next to the .dsc"): resolve_dsc_files(str(dsc_path)) -def test_resolve_dsc_files_no_source_tarball_errors(tmp_path): - _touch(tmp_path, "pkg_1.0-1_amd64.changes") - dsc_path = _write_dsc(tmp_path, ["pkg_1.0-1_amd64.changes"]) +def test_rejects_symlink_that_escapes_dsc_directory(tmp_path): + dsc_dir = tmp_path / "source-package" + dsc_dir.mkdir() + outside_archive = _touch(tmp_path, "outside.tar.gz") + (dsc_dir / "pkg_1.0.tar.gz").symlink_to(outside_archive) + dsc_path = _write_dsc(dsc_dir, ["pkg_1.0.tar.gz"]) - with pytest.raises(click.UsageError, match="does not reference a source tarball"): + with pytest.raises(click.UsageError, match="not a regular file directly"): resolve_dsc_files(str(dsc_path)) + + +@pytest.mark.parametrize( + ("source", "version", "filename"), + [ + ("foo.orig-bar", "1.0", "foo.orig-bar_1.0.tar.gz"), + ("foo", "1.0.orig-bar", "foo_1.0.orig-bar.tar.gz"), + ], +) +def test_native_name_or_version_containing_orig_marker_is_not_a_component( + source, version, filename, tmp_path +): + source_archive = _touch(tmp_path, filename) + dsc_path = _write_dsc( + tmp_path, [filename], source=source, version=version, name="package.dsc" + ) + + assert resolve_dsc_files(str(dsc_path)) == (str(source_archive), None) + + +def test_rejects_missing_file_listing(tmp_path): + dsc_path = tmp_path / "empty.dsc" + dsc_path.write_text("Format: 3.0 (native)\nSource: pkg\nVersion: 1.0\n") + + with pytest.raises(click.UsageError, match="Checksums-Sha256"): + resolve_dsc_files(str(dsc_path)) + + +def test_resolved_paths_are_canonical(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.tar.gz") + internal_link = tmp_path / "linked" + internal_link.symlink_to(tmp_path, target_is_directory=True) + dsc_path = _write_dsc(tmp_path, [source_archive.name]) + + sources_file, _ = resolve_dsc_files(os.path.join(str(internal_link), dsc_path.name)) + + assert sources_file == str(source_archive) diff --git a/cloudsmith_cli/cli/tests/test_push_dsc_cli.py b/cloudsmith_cli/cli/tests/test_push_dsc_cli.py index 5f30c0cc..f94eba30 100644 --- a/cloudsmith_cli/cli/tests/test_push_dsc_cli.py +++ b/cloudsmith_cli/cli/tests/test_push_dsc_cli.py @@ -1,17 +1,12 @@ -"""CLI-level tests for `cloudsmith push deb --dsc-file` (GitHub issue #56). - -Complements ``test_dsc_parser.py`` (pure ``.dsc`` parsing/resolution) with -integration coverage of the ``deb`` push handler: the flag itself, the ---sources-file/--changes-file precedence rule, and that a rejected .dsc -aborts before any network call. -""" +"""Registered-command tests for ``cloudsmith push deb --dsc-file``.""" +from pathlib import Path from unittest.mock import patch import pytest from .. import config as cli_config -from ..commands.push import push +from ..commands.main import main HERMETIC_ARGS = ["--api-key", "fake-api-key"] @@ -26,7 +21,7 @@ @pytest.fixture(autouse=True) def hermetic_environment(monkeypatch): - """Keep a developer's real env/config out of these tests (see test_domains.py).""" + """Keep developer environment and config values out of command tests.""" monkeypatch.delenv("CLOUDSMITH_ORG", raising=False) monkeypatch.delenv("CLOUDSMITH_CONFIG_FILE", raising=False) monkeypatch.delattr(cli_config.OPTIONS, "value", raising=False) @@ -40,30 +35,67 @@ def _touch(tmp_path, name): return path -def _write_dsc(tmp_path, file_lines, field="Files", name="pkg_1.0-1.dsc"): - body = "\n".join(f" deadbeef 100 {line}" for line in file_lines) - dsc_path = tmp_path / name - dsc_path.write_text( - f"Format: 3.0 (native)\nSource: pkg\nVersion: 1.0-1\n{field}:\n{body}\n" +def _file_field(name, filenames): + checksum = "a" * (64 if name == "Checksums-Sha256" else 32) + entries = "\n".join(f" {checksum} 100 {filename}" for filename in filenames) + return f"{name}:\n{entries}\n" + + +def _write_dsc( + tmp_path, + filenames, + *, + source="pkg", + version="1.0", + source_format="3.0 (native)", + clearsigned=False, +): + control = ( + f"Format: {source_format}\nSource: {source}\nVersion: {version}\n" + + _file_field("Files", filenames) + + _file_field("Checksums-Sha256", filenames) ) + if clearsigned: + control = ( + "-----BEGIN PGP SIGNED MESSAGE-----\n" + "Hash: SHA256\n\n" + f"{control}" + "-----BEGIN PGP SIGNATURE-----\n" + "test-signature-data\n" + "-----END PGP SIGNATURE-----\n" + ) + dsc_path = tmp_path / f"{source}_{version}.dsc" + dsc_path.write_text(control) return dsc_path +def _upload_identifier(*, filepath, **_kwargs): + return f"uploaded:{Path(filepath).name}" + + def _invoke(runner, tmp_path, extra_args, mocks): package_file = _touch(tmp_path, "pkg_1.0-1_amd64.deb") with ( patch(_MOCK_TARGETS[0]) as mock_validate_create_package, - patch(_MOCK_TARGETS[1], return_value="checksum"), - patch(_MOCK_TARGETS[2], return_value="file-id"), - patch(_MOCK_TARGETS[3], return_value=("slug-perm", "slug")), + patch(_MOCK_TARGETS[1], return_value="checksum") as mock_validate_upload, + patch(_MOCK_TARGETS[2], side_effect=_upload_identifier) as mock_upload, + patch( + _MOCK_TARGETS[3], return_value=("slug-perm", "slug") + ) as mock_create_package, patch(_MOCK_TARGETS[4]), ): - mocks["validate_create_package"] = mock_validate_create_package + mocks.update( + validate_create_package=mock_validate_create_package, + validate_upload_file=mock_validate_upload, + upload_file=mock_upload, + create_package=mock_create_package, + ) result = runner.invoke( - push, + main, [ + "push", "deb", - "acme/repo/ubuntu/xenial", + "example/repo/ubuntu/xenial", str(package_file), *extra_args, *HERMETIC_ARGS, @@ -73,26 +105,76 @@ def _invoke(runner, tmp_path, extra_args, mocks): return result -def test_dsc_file_derives_sources_and_changes_files(runner, tmp_path): - sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") - changes_file = _touch(tmp_path, "pkg_1.0-1_amd64.changes") - dsc_path = _write_dsc(tmp_path, [sources_tarball.name, changes_file.name]) +@pytest.mark.parametrize( + ("source_format", "version", "source_name", "changes_name"), + [ + ("3.0 (native)", "1.0", "pkg_1.0.tar.xz", None), + ( + "3.0 (quilt)", + "1.0-1", + "pkg_1.0.orig.tar.gz", + "pkg_1.0-1.debian.tar.xz", + ), + ("1.0", "1.0-1", "pkg_1.0.orig.tar.gz", "pkg_1.0-1.diff.gz"), + ], +) +def test_dsc_maps_real_source_package_members_to_uploaded_sdk_fields( + runner, tmp_path, source_format, version, source_name, changes_name +): + source_archive = _touch(tmp_path, source_name) + filenames = [source_archive.name] + if changes_name: + filenames.append(_touch(tmp_path, changes_name).name) + dsc_path = _write_dsc( + tmp_path, filenames, version=version, source_format=source_format + ) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + + assert result.exit_code == 0, result.output + validation_kwargs = mocks["validate_create_package"].call_args.kwargs + assert validation_kwargs["sources_file"] == str(source_archive) + assert validation_kwargs["changes_file"] == ( + str(tmp_path / changes_name) if changes_name else None + ) + create_kwargs = mocks["create_package"].call_args.kwargs + assert create_kwargs["sources_file"] == f"uploaded:{source_name}" + assert create_kwargs["changes_file"] == ( + f"uploaded:{changes_name}" if changes_name else None + ) + + +def test_clearsigned_dsc_is_parsed_through_registered_command(runner, tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name, debian_archive.name], + version="1.0-1", + source_format="3.0 (quilt)", + clearsigned=True, + ) mocks = {} result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) assert result.exit_code == 0, result.output kwargs = mocks["validate_create_package"].call_args.kwargs - assert kwargs["sources_file"] == str(sources_tarball) - assert kwargs["changes_file"] == str(changes_file) + assert kwargs["sources_file"] == str(source_archive) + assert kwargs["changes_file"] == str(debian_archive) -def test_explicit_sources_file_wins_over_dsc_file(runner, tmp_path): - """Explicit --sources-file always takes precedence over --dsc-file.""" - dsc_sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") - changes_file = _touch(tmp_path, "pkg_1.0-1_amd64.changes") - explicit_sources_file = _touch(tmp_path, "explicit_sources.tar.gz") - dsc_path = _write_dsc(tmp_path, [dsc_sources_tarball.name, changes_file.name]) +def test_explicit_sources_file_wins_per_field(runner, tmp_path): + dsc_source = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + explicit_source = _touch(tmp_path, "explicit.tar.gz") + dsc_path = _write_dsc( + tmp_path, + [dsc_source.name, debian_archive.name], + version="1.0-1", + source_format="3.0 (quilt)", + ) mocks = {} result = _invoke( @@ -102,25 +184,27 @@ def test_explicit_sources_file_wins_over_dsc_file(runner, tmp_path): "--dsc-file", str(dsc_path), "--sources-file", - str(explicit_sources_file), + str(explicit_source), ], mocks, ) assert result.exit_code == 0, result.output kwargs = mocks["validate_create_package"].call_args.kwargs - # Explicit flag wins... - assert kwargs["sources_file"] == str(explicit_sources_file) - # ...but changes_file, which was not passed explicitly, is still derived. - assert kwargs["changes_file"] == str(changes_file) + assert kwargs["sources_file"] == str(explicit_source) + assert kwargs["changes_file"] == str(debian_archive) -def test_explicit_changes_file_wins_over_dsc_file(runner, tmp_path): - """Explicit --changes-file always takes precedence over --dsc-file.""" - sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") - dsc_changes_file = _touch(tmp_path, "pkg_1.0-1_amd64.changes") - explicit_changes_file = _touch(tmp_path, "explicit.changes") - dsc_path = _write_dsc(tmp_path, [sources_tarball.name, dsc_changes_file.name]) +def test_explicit_changes_file_wins_per_field(runner, tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + dsc_changes = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + explicit_changes = _touch(tmp_path, "explicit.diff.gz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name, dsc_changes.name], + version="1.0-1", + source_format="3.0 (quilt)", + ) mocks = {} result = _invoke( @@ -130,21 +214,55 @@ def test_explicit_changes_file_wins_over_dsc_file(runner, tmp_path): "--dsc-file", str(dsc_path), "--changes-file", - str(explicit_changes_file), + str(explicit_changes), ], mocks, ) assert result.exit_code == 0, result.output kwargs = mocks["validate_create_package"].call_args.kwargs - assert kwargs["sources_file"] == str(sources_tarball) - assert kwargs["changes_file"] == str(explicit_changes_file) + assert kwargs["sources_file"] == str(source_archive) + assert kwargs["changes_file"] == str(explicit_changes) + + +@pytest.mark.parametrize("member", ["/etc/passwd", "../pkg_1.0.tar.gz"]) +def test_member_path_is_rejected_before_network_calls(runner, tmp_path, member): + dsc_path = _write_dsc(tmp_path, [member]) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + + assert result.exit_code != 0 + assert "must be filenames next to the .dsc" in result.output + mocks["validate_create_package"].assert_not_called() + mocks["validate_upload_file"].assert_not_called() -def test_dsc_file_rejects_multi_component_before_any_network_call(runner, tmp_path): - orig_tarball = _touch(tmp_path, "pkg_1.0.orig.tar.gz") - component_tarball = _touch(tmp_path, "pkg_1.0.orig-libbar.tar.gz") - dsc_path = _write_dsc(tmp_path, [orig_tarball.name, component_tarball.name]) +def test_escaping_symlink_is_rejected_before_network_calls(runner, tmp_path): + dsc_dir = tmp_path / "dsc" + dsc_dir.mkdir() + outside_archive = _touch(tmp_path, "outside.tar.gz") + (dsc_dir / "pkg_1.0.tar.gz").symlink_to(outside_archive) + dsc_path = _write_dsc(dsc_dir, ["pkg_1.0.tar.gz"]) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + + assert result.exit_code != 0 + assert "not a regular file directly next to the .dsc" in result.output + mocks["validate_create_package"].assert_not_called() + mocks["validate_upload_file"].assert_not_called() + + +def test_multi_component_is_rejected_before_network_calls(runner, tmp_path): + filenames = [ + _touch(tmp_path, "pkg_1.0.orig.tar.gz").name, + _touch(tmp_path, "pkg_1.0.orig-libbar.tar.gz").name, + _touch(tmp_path, "pkg_1.0-1.debian.tar.xz").name, + ] + dsc_path = _write_dsc( + tmp_path, filenames, version="1.0-1", source_format="3.0 (quilt)" + ) mocks = {} result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) @@ -154,28 +272,34 @@ def test_dsc_file_rejects_multi_component_before_any_network_call(runner, tmp_pa mocks["validate_create_package"].assert_not_called() -def test_dsc_file_rejects_detached_signature_before_any_network_call(runner, tmp_path): - sources_tarball = _touch(tmp_path, "pkg_1.0.tar.gz") - signature_file = _touch(tmp_path, "pkg_1.0.tar.gz.asc") - dsc_path = _write_dsc(tmp_path, [sources_tarball.name, signature_file.name]) +def test_detached_signature_is_rejected_before_network_calls(runner, tmp_path): + filenames = [ + _touch(tmp_path, "pkg_1.0.orig.tar.gz").name, + _touch(tmp_path, "pkg_1.0.orig.tar.gz.asc").name, + _touch(tmp_path, "pkg_1.0-1.debian.tar.xz").name, + ] + dsc_path = _write_dsc( + tmp_path, filenames, version="1.0-1", source_format="3.0 (quilt)" + ) mocks = {} result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) assert result.exit_code != 0 - assert "detached signature" in result.output + assert "detached upstream signature" in result.output mocks["validate_create_package"].assert_not_called() def test_push_deb_help_documents_dsc_file_option(runner): - result = runner.invoke(push, ["deb", "--help"], catch_exceptions=False) + result = runner.invoke(main, ["push", "deb", "--help"], catch_exceptions=False) assert result.exit_code == 0, result.output assert "--dsc-file" in result.output + assert ".debian.tar.*" in result.output def test_push_non_deb_format_has_no_dsc_file_option(runner): - result = runner.invoke(push, ["raw", "--help"], catch_exceptions=False) + result = runner.invoke(main, ["push", "raw", "--help"], catch_exceptions=False) assert result.exit_code == 0, result.output assert "--dsc-file" not in result.output From 78a6f2fbb73a75630b61f212f4c8fb18e56d7818 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Thu, 13 Aug 2026 16:32:48 +0100 Subject: [PATCH 3/4] fix(ENG-56): derive `.dsc` members by default and accept real packages Verified against the live API, a Debian source push must name the `.dsc` as PACKAGE_FILE: a binary `.deb` carrying source archives is rejected ("You can only upload a sources and changes files with a source package") and a `.dsc` on its own is rejected too ("A sources archive is required when uploading a source package"). Requiring `--dsc-file` as well therefore meant writing the same path twice, where issue #56 asks for the members to be derived "when passed only a .dsc file". The `.dsc` is now parsed whenever PACKAGE_FILE ends in `.dsc` and no `--sources-file` was given; `--dsc-file` remains for naming a different `.dsc`. Since a `.dsc`-only push already failed server-side, this cannot regress a working invocation. Fixes found while testing against real packages: - Detached upstream signatures hard-failed, which made the feature unusable for Debian's own `hello` source package. The issue notes signatures can be ignored, so they are now skipped and reported via `utils.should_use_stderr(opts)`; `resolve_dsc_files()` returns a frozen `ResolvedDscFiles` carrying `ignored_files` so the parser reports nothing itself, mirroring `ResolvedMetadata`. - A signature also matched as a second source archive, since `pkg.orig.tar.gz.asc` satisfies the `pkg.orig.tar.` prefix test. Masked by the hard failure above; classification now runs over the non-signature members only. - `3.0 (git)` and `3.0 (bzr)` were declared supported but could never resolve: per dpkg-source(1) and Dpkg::Source::Package::V3::Bzr they produce a git bundle and a `.bzr.tar.*`, neither matching `_.tar.*`, and neither being a source archive the deb upload model can index. Both now fall through to the unsupported-format error alongside `3.0 (custom)`. - Symlinked members were rejected because the resolved target had to sit in the `.dsc` directory. `mk-origtargz --symlink` is the uscan default and links the `.orig` tarball in from a download cache. Traversal is already blocked at the name level, so the check is now "regular file next to the .dsc", still returning the canonical path so a symlink cannot be swapped between check and read. A member symlinked to a directory is still rejected. - Errors named `--dsc-file` even when the `.dsc` came from PACKAGE_FILE, and now lead with the file. - `--dsc-file` used `click.Path` where every other file option in `push.py` uses `ExpandPath`. Verified end-to-end against a live repository with `hello_2.10-3` (`3.0 (quilt)`, clearsigned, ships a `.asc`), a `3.0 (native)` package and a `1.0` non-native package with a `.diff.gz`; all three synchronised with the expected members stored. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 2 +- CHANGELOG.md | 4 +- README.md | 6 ++ cloudsmith_cli/cli/commands/push.py | 62 +++++++++--- cloudsmith_cli/cli/dsc_parser.py | 94 ++++++++++++------- cloudsmith_cli/cli/tests/test_dsc_parser.py | 65 ++++++++----- cloudsmith_cli/cli/tests/test_push_dsc_cli.py | 85 +++++++++++++++-- 7 files changed, 236 insertions(+), 82 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 63edb6cf..38a67757 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ The CLI is a Click app that wraps the auto-generated `cloudsmith-api` Python SDK ### Push + metadata coupling -`cli/commands/push.py` is the most complex command. Every `push ` subcommand accepts `--metadata-*` flags resolved via `metadata_common`. Push validates metadata both locally and against the API **before** any file upload so malformed SBOM/BuildInfo payloads cannot leave orphan packages behind. Failure behavior is configurable with precedence: `--on-metadata-failure` flag > `$CLOUDSMITH_METADATA_FAILURE_MODE` env > `metadata_failure_mode` config key > `error` default. The kwarg names listed in `METADATA_KWARG_NAMES` and `METADATA_FAILURE_MODE_KWARG` must be popped off the kwargs before they reach the API client, which will reject unknown keys. +`cli/commands/push.py` is the most complex command. The per-format subcommands and their options are generated from the API's package-format model, so format-specific client-side behaviour is layered on top of that loop rather than branching inside it — see the `deb`-only `--dsc-file` registration at the end of `create_push_handlers()`, which pairs with `cli/dsc_parser.py` to derive `sources_file`/`changes_file` from a Debian `.dsc`. Every `push ` subcommand accepts `--metadata-*` flags resolved via `metadata_common`. Push validates metadata both locally and against the API **before** any file upload so malformed SBOM/BuildInfo payloads cannot leave orphan packages behind. Failure behavior is configurable with precedence: `--on-metadata-failure` flag > `$CLOUDSMITH_METADATA_FAILURE_MODE` env > `metadata_failure_mode` config key > `error` default. The kwarg names listed in `METADATA_KWARG_NAMES` and `METADATA_FAILURE_MODE_KWARG` must be popped off the kwargs before they reach the API client, which will reject unknown keys. ### Output formatting convention diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f87991..8f17ec28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] -## [1.22.0] - 2026-08-11 +### Added + +- `cloudsmith push deb` now derives a Debian source package's members from its `.dsc`, so `cloudsmith push deb /// foo_1.0-1.dsc` is enough where `--sources-file` and `--changes-file` previously had to be worked out by hand (and their suffixes vary: `.orig.tar.gz`, `.orig.tar.bz2`, `.debian.tar.xz`, `.diff.gz`, ...). The `Checksums-Sha256:` or `Files:` field of the `.dsc` is read — plain or OpenPGP-clearsigned — and the upstream/native source archive becomes `--sources-file` while the Debian packaging archive becomes `--changes-file`, for the `1.0`, `2.0`, `3.0 (native)` and `3.0 (quilt)` source formats. `--dsc-file` names a `.dsc` other than `PACKAGE_FILE`, and an explicit `--sources-file` or `--changes-file` still wins for its own field. A detached upstream signature (`*.orig.tar.*.asc`) is skipped with a warning, since the deb package format has no field to carry it; a multi-component source package (`*.orig-.tar.*`) is rejected outright, because leaving a component behind would upload incomplete source. ### Added diff --git a/README.md b/README.md index 545edfb7..b6463c66 100644 --- a/README.md +++ b/README.md @@ -363,6 +363,12 @@ For example, if you wanted to upload a Debian package, you can do it in one-step cloudsmith push deb your-org/your-repo/ubuntu/xenial libxml2-2.9.4-2.x86_64.deb ``` +To upload a Debian *source* package, give it the `.dsc`. The source and Debian packaging archives listed in the `.dsc` are found and uploaded with it: + +``` +cloudsmith push deb your-org/your-repo/ubuntu/xenial hello_2.10-3.dsc +``` + Want to know how to do it with another packaging format? Easy, just ask for help: ``` diff --git a/cloudsmith_cli/cli/commands/push.py b/cloudsmith_cli/cli/commands/push.py index 786c8cf0..1dbe1b30 100644 --- a/cloudsmith_cli/cli/commands/push.py +++ b/cloudsmith_cli/cli/commands/push.py @@ -76,6 +76,10 @@ #: separately from the metadata payload kwargs so it does not leak into the #: package-create API call. METADATA_FAILURE_MODE_KWARG = "cli_metadata_failure_mode" +#: Filename suffix of a Debian source control file. A ``deb`` push whose +#: PACKAGE_FILE carries it is a source-package upload, so its members can be +#: derived without ``--dsc-file`` (GitHub issue #56). +DSC_SUFFIX = ".dsc" def _metadata_failure_is_warn(opts=None): @@ -1020,6 +1024,13 @@ def upload_files_and_create_package( return slug_perm, slug +def _implied_dsc_file(package_file): + """Return ``package_file`` if it is itself a Debian ``.dsc``, else None.""" + if isinstance(package_file, str) and package_file.endswith(DSC_SUFFIX): + return package_file + return None + + def create_push_handlers(): """Create a handler for upload per package format.""" # pylint: disable=fixme @@ -1187,16 +1198,32 @@ def push_handler(ctx, *args, **kwargs): # `deb` subcommand ever registers --dsc-file (below), so this is # a no-op kwargs.pop() for every other format. dsc_file = kwargs.pop("dsc_file", None) + if dsc_file is None and not kwargs.get("sources_file"): + # Pushing a Debian source package means passing the .dsc as + # PACKAGE_FILE, and the API rejects it without a sources + # archive, so parse it by default rather than making the user + # name the same file twice. An explicit --sources-file means + # the caller is driving the members manually; leave them to it. + dsc_file = _implied_dsc_file(kwargs.get("package_file")) if dsc_file: - dsc_sources_file, dsc_changes_file = resolve_dsc_files(dsc_file) + resolved_dsc = resolve_dsc_files(dsc_file) + if resolved_dsc.ignored_files: + click.secho( + "Not uploading {files}: the deb package format has no " + "field for detached upstream signatures.".format( + files=", ".join(resolved_dsc.ignored_files) + ), + fg="yellow", + err=utils.should_use_stderr(opts), + ) # Precedence: explicit --sources-file/--changes-file always # win over values derived from --dsc-file. A user who passes # both wants a manual override, not to have their explicit # flag silently clobbered. if not kwargs.get("sources_file"): - kwargs["sources_file"] = dsc_sources_file + kwargs["sources_file"] = resolved_dsc.sources_file if not kwargs.get("changes_file"): - kwargs["changes_file"] = dsc_changes_file + kwargs["changes_file"] = resolved_dsc.changes_file owner_repo = kwargs.pop("owner_repo") if "distribution" in parameters: @@ -1332,23 +1359,28 @@ def push_handler(ctx, *args, **kwargs): push_handler = click.option( "--dsc-file", "dsc_file", - type=click.Path( - exists=True, dir_okay=False, readable=True, resolve_path=True + type=ExpandPath( + dir_okay=False, exists=True, writable=False, resolve_path=True ), default=None, help=( - "Path to a Debian .dsc control file. Its 'Files:' (or " - "'Checksums-Sha256:') field is parsed to auto-fill " + "Path to a Debian .dsc control file, plain or " + "OpenPGP-clearsigned. Only needed to point at a .dsc " + "other than PACKAGE_FILE: pushing a source package means " + "passing the .dsc as PACKAGE_FILE, and that is parsed " + "automatically unless --sources-file is given. Its " + "'Checksums-Sha256:' (or 'Files:') field supplies " "--sources-file with the upstream/native source archive " "and --changes-file with the Debian packaging archive " - "(.debian.tar.* or .diff.gz). Referenced files must be " - "directly next to the .dsc. Any " - "--sources-file/--changes-file passed explicitly always " - "takes precedence over a value derived from --dsc-file. " - "Rejected with an error if the .dsc references a " - "multi-component source package (*.orig-*.tar.*) or a " - "detached signature (*.asc) -- Cloudsmith's deb upload " - "format does not support either." + "(.debian.tar.* or .diff.gz). Supports the 1.0, 2.0, " + "3.0 (native) and 3.0 (quilt) source formats, and the " + "referenced files must sit next to the .dsc. An explicit " + "--sources-file or --changes-file always takes precedence " + "over the derived value for that field. Detached upstream " + "signatures (*.asc) are skipped with a warning; a " + "multi-component source package (*.orig-*.tar.*) is " + "rejected, because the deb package format has no field " + "for the extra components." ), )(push_handler) diff --git a/cloudsmith_cli/cli/dsc_parser.py b/cloudsmith_cli/cli/dsc_parser.py index 1f6d1ab8..0e8ee3e6 100644 --- a/cloudsmith_cli/cli/dsc_parser.py +++ b/cloudsmith_cli/cli/dsc_parser.py @@ -1,6 +1,7 @@ -"""Parse Debian ``.dsc`` control files for ``push deb --dsc-file``.""" +"""Parse Debian ``.dsc`` control files for ``push deb`` source uploads.""" import os +from dataclasses import dataclass, field from email.parser import Parser import click @@ -10,11 +11,23 @@ _DETACHED_SIGNATURE_SUFFIX = ".asc" _FILE_LIST_FIELDS = ("Checksums-Sha256", "Files") _QUILT_FORMATS = {"2.0", "3.0 (quilt)"} -_SINGLE_TARBALL_FORMATS = {"3.0 (native)", "3.0 (git)", "3.0 (bzr)"} +_NATIVE_FORMAT = "3.0 (native)" + + +@dataclass(frozen=True) +class ResolvedDscFiles: + """Source-package members resolved from a Debian ``.dsc``.""" + + sources_file: str + changes_file: str | None = None + #: Members deliberately left out of the upload, for the caller to report. + ignored_files: tuple[str, ...] = field(default_factory=tuple) def _usage_error(dsc_path, message): - return click.UsageError(f"--dsc-file {dsc_path!r} {message}") + # Names the file rather than the option: the .dsc is just as often taken + # from PACKAGE_FILE as from an explicit --dsc-file. + return click.UsageError(f"Debian source control file {dsc_path!r} {message}") def _unwrap_clearsigned_control(text, dsc_path): @@ -127,19 +140,19 @@ def _classify_members(message, filenames, dsc_path): source_format = _required_field(message, "Format", dsc_path) upstream_version = version.rsplit("-", 1)[0] + # Detached upstream signatures (e.g. hello_2.10.orig.tar.gz.asc) are + # common and have no field in the deb upload model. Nothing in the + # uploaded source is lost by leaving them out, so they are skipped with a + # warning rather than failing the push. signature_files = [ filename for filename in filenames if filename.endswith(_DETACHED_SIGNATURE_SUFFIX) ] - if signature_files: - raise _usage_error( - dsc_path, - "references detached upstream signature file(s) ({files}). Cloudsmith " - "does not support detached upstream signatures for deb uploads.".format( - files=", ".join(signature_files) - ), - ) + # Classify the uploadable members only. A signature shares its tarball's + # stem (`*.orig.tar.gz.asc`), so leaving it in would match as a second + # source archive. + members = [f for f in filenames if f not in set(signature_files)] sources = [] changes = [] @@ -150,29 +163,33 @@ def _classify_members(message, filenames, dsc_path): orig_stem = f"{source}_{upstream_version}.orig" component_prefix = f"{orig_stem}-" debian_stem = f"{source}_{version}.debian" - sources = [f for f in filenames if _is_tar_archive(f, orig_stem)] - changes = [f for f in filenames if _is_tar_archive(f, debian_stem)] + sources = [f for f in members if _is_tar_archive(f, orig_stem)] + changes = [f for f in members if _is_tar_archive(f, debian_stem)] component_files = [ f - for f in filenames + for f in members if f.startswith(component_prefix) and ".tar." in f[len(component_prefix) :] ] elif source_format == "1.0": orig_stem = f"{source}_{upstream_version}.orig" diff_name = f"{source}_{version}.diff.gz" native_stem = f"{source}_{version}" - orig_files = [f for f in filenames if _is_tar_archive(f, orig_stem)] - diff_files = [f for f in filenames if f == diff_name] - native_files = [f for f in filenames if _is_tar_archive(f, native_stem)] + orig_files = [f for f in members if _is_tar_archive(f, orig_stem)] + diff_files = [f for f in members if f == diff_name] + native_files = [f for f in members if _is_tar_archive(f, native_stem)] if orig_files or diff_files: legacy_non_native = True sources = orig_files changes = diff_files else: sources = native_files - elif source_format in _SINGLE_TARBALL_FORMATS: - sources = [f for f in filenames if _is_tar_archive(f, f"{source}_{version}")] + elif source_format == _NATIVE_FORMAT: + sources = [f for f in members if _is_tar_archive(f, f"{source}_{version}")] else: + # '3.0 (git)' ships a git bundle and '3.0 (bzr)' a VCS tarball, neither + # of which is a source archive the deb upload model can index, so both + # fall through to the unsupported-format error alongside + # '3.0 (custom)'. raise _usage_error( dsc_path, f"uses unsupported Debian source format {source_format!r}." ) @@ -185,8 +202,8 @@ def _classify_members(message, filenames, dsc_path): "uploads.".format(files=", ".join(component_files)), ) - classified = set(sources + changes + signature_files + component_files) - unexpected = [f for f in filenames if f not in classified] + classified = set(sources + changes + component_files) + unexpected = [f for f in members if f not in classified] if unexpected: raise _usage_error( dsc_path, @@ -212,33 +229,40 @@ def _classify_members(message, filenames, dsc_path): f"format '1.0'; found {len(changes)}.", ) - return sources[0], changes[0] if changes else None + return sources[0], changes[0] if changes else None, tuple(signature_files) def _resolve_member(base_dir, filename, dsc_path): + """Canonicalise a member that ``_validate_member_names`` has vetted.""" candidate = os.path.join(base_dir, filename) - resolved = os.path.realpath(candidate) - if os.path.dirname(resolved) != base_dir or not os.path.isfile(resolved): + if not os.path.isfile(candidate): raise _usage_error( dsc_path, - f"references {filename!r}, but it is not a regular file directly " - "next to the .dsc.", + f"references {filename!r}, but it is not a regular file next to the .dsc.", ) - return resolved + # Uploading the canonical path means a symlink swapped between this check + # and the upload cannot redirect the read. The link itself may point + # outside the directory: `mk-origtargz --symlink` (the uscan default) + # routinely symlinks the .orig tarball in from a download cache. + return os.path.realpath(candidate) def resolve_dsc_files(dsc_path): - """Return ``(sources_file, changes_file)`` derived from a Debian ``.dsc``.""" + """Return the :class:`ResolvedDscFiles` derived from a Debian ``.dsc``.""" message = _read_control_message(dsc_path) filenames = _extract_filenames(message, dsc_path) _validate_member_names(filenames, dsc_path) - source_filename, changes_filename = _classify_members(message, filenames, dsc_path) + source_filename, changes_filename, ignored_files = _classify_members( + message, filenames, dsc_path + ) base_dir = os.path.realpath(os.path.dirname(os.path.abspath(dsc_path))) - sources_file = _resolve_member(base_dir, source_filename, dsc_path) - changes_file = ( - _resolve_member(base_dir, changes_filename, dsc_path) - if changes_filename - else None + return ResolvedDscFiles( + sources_file=_resolve_member(base_dir, source_filename, dsc_path), + changes_file=( + _resolve_member(base_dir, changes_filename, dsc_path) + if changes_filename + else None + ), + ignored_files=ignored_files, ) - return sources_file, changes_file diff --git a/cloudsmith_cli/cli/tests/test_dsc_parser.py b/cloudsmith_cli/cli/tests/test_dsc_parser.py index 6b28d2a9..3722d7d8 100644 --- a/cloudsmith_cli/cli/tests/test_dsc_parser.py +++ b/cloudsmith_cli/cli/tests/test_dsc_parser.py @@ -5,7 +5,7 @@ import click import pytest -from ..dsc_parser import resolve_dsc_files +from ..dsc_parser import ResolvedDscFiles, resolve_dsc_files def _file_field(name, filenames): @@ -53,7 +53,7 @@ def test_resolves_native_source_archive(tmp_path): source_archive = _touch(tmp_path, "pkg_1.0.tar.xz") dsc_path = _write_dsc(tmp_path, [source_archive.name]) - assert resolve_dsc_files(str(dsc_path)) == (str(source_archive), None) + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles(str(source_archive)) def test_resolves_quilt_source_and_debian_archives(tmp_path): @@ -66,9 +66,8 @@ def test_resolves_quilt_source_and_debian_archives(tmp_path): source_format="3.0 (quilt)", ) - assert resolve_dsc_files(str(dsc_path)) == ( - str(source_archive), - str(debian_archive), + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles( + str(source_archive), str(debian_archive) ) @@ -82,9 +81,8 @@ def test_resolves_legacy_non_native_source_and_diff_archives(tmp_path): source_format="1.0", ) - assert resolve_dsc_files(str(dsc_path)) == ( - str(source_archive), - str(diff_archive), + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles( + str(source_archive), str(diff_archive) ) @@ -113,9 +111,8 @@ def test_resolves_clearsigned_dsc_using_matching_sha256_and_files(tmp_path): clearsigned=True, ) - assert resolve_dsc_files(str(dsc_path)) == ( - str(source_archive), - str(debian_archive), + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles( + str(source_archive), str(debian_archive) ) @@ -123,7 +120,7 @@ def test_uses_checksums_sha256_when_files_is_absent(tmp_path): source_archive = _touch(tmp_path, "pkg_1.0.tar.gz") dsc_path = _write_dsc(tmp_path, [source_archive.name], fields=("Checksums-Sha256",)) - assert resolve_dsc_files(str(dsc_path)) == (str(source_archive), None) + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles(str(source_archive)) def test_rejects_conflicting_checksum_file_lists(tmp_path): @@ -152,7 +149,7 @@ def test_rejects_malformed_file_list_row(tmp_path): def test_rejects_missing_referenced_file(tmp_path): dsc_path = _write_dsc(tmp_path, ["pkg_1.0.tar.gz"]) - with pytest.raises(click.UsageError, match="not a regular file directly"): + with pytest.raises(click.UsageError, match="not a regular file next to"): resolve_dsc_files(str(dsc_path)) @@ -175,10 +172,10 @@ def test_rejects_multi_component_source_package(tmp_path): resolve_dsc_files(str(dsc_path)) -def test_rejects_detached_signature(tmp_path): - _touch(tmp_path, "pkg_1.0.orig.tar.gz") +def test_skips_detached_signature_without_failing(tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") _touch(tmp_path, "pkg_1.0.orig.tar.gz.asc") - _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") dsc_path = _write_dsc( tmp_path, [ @@ -190,7 +187,19 @@ def test_rejects_detached_signature(tmp_path): source_format="3.0 (quilt)", ) - with pytest.raises(click.UsageError, match="detached upstream signature"): + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles( + str(source_archive), + str(debian_archive), + ignored_files=("pkg_1.0.orig.tar.gz.asc",), + ) + + +@pytest.mark.parametrize("source_format", ["3.0 (git)", "3.0 (bzr)", "3.0 (custom)"]) +def test_rejects_source_formats_without_an_indexable_archive(source_format, tmp_path): + _touch(tmp_path, "pkg_1.0.git") + dsc_path = _write_dsc(tmp_path, ["pkg_1.0.git"], source_format=source_format) + + with pytest.raises(click.UsageError, match="unsupported Debian source format"): resolve_dsc_files(str(dsc_path)) @@ -202,14 +211,26 @@ def test_rejects_member_paths(filename, tmp_path): resolve_dsc_files(str(dsc_path)) -def test_rejects_symlink_that_escapes_dsc_directory(tmp_path): +def test_resolves_symlinked_member_to_its_canonical_target(tmp_path): + # `mk-origtargz --symlink` (the uscan default) links the .orig tarball in + # from a download cache, so a member pointing outside the .dsc directory + # is a normal build tree, not an attempt to escape it. dsc_dir = tmp_path / "source-package" dsc_dir.mkdir() outside_archive = _touch(tmp_path, "outside.tar.gz") (dsc_dir / "pkg_1.0.tar.gz").symlink_to(outside_archive) dsc_path = _write_dsc(dsc_dir, ["pkg_1.0.tar.gz"]) - with pytest.raises(click.UsageError, match="not a regular file directly"): + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles(str(outside_archive)) + + +def test_rejects_member_symlinked_to_a_directory(tmp_path): + dsc_dir = tmp_path / "source-package" + dsc_dir.mkdir() + (dsc_dir / "pkg_1.0.tar.gz").symlink_to(tmp_path, target_is_directory=True) + dsc_path = _write_dsc(dsc_dir, ["pkg_1.0.tar.gz"]) + + with pytest.raises(click.UsageError, match="not a regular file next to"): resolve_dsc_files(str(dsc_path)) @@ -228,7 +249,7 @@ def test_native_name_or_version_containing_orig_marker_is_not_a_component( tmp_path, [filename], source=source, version=version, name="package.dsc" ) - assert resolve_dsc_files(str(dsc_path)) == (str(source_archive), None) + assert resolve_dsc_files(str(dsc_path)) == ResolvedDscFiles(str(source_archive)) def test_rejects_missing_file_listing(tmp_path): @@ -245,6 +266,6 @@ def test_resolved_paths_are_canonical(tmp_path): internal_link.symlink_to(tmp_path, target_is_directory=True) dsc_path = _write_dsc(tmp_path, [source_archive.name]) - sources_file, _ = resolve_dsc_files(os.path.join(str(internal_link), dsc_path.name)) + resolved = resolve_dsc_files(os.path.join(str(internal_link), dsc_path.name)) - assert sources_file == str(source_archive) + assert resolved.sources_file == str(source_archive) diff --git a/cloudsmith_cli/cli/tests/test_push_dsc_cli.py b/cloudsmith_cli/cli/tests/test_push_dsc_cli.py index f94eba30..12f2c544 100644 --- a/cloudsmith_cli/cli/tests/test_push_dsc_cli.py +++ b/cloudsmith_cli/cli/tests/test_push_dsc_cli.py @@ -73,8 +73,9 @@ def _upload_identifier(*, filepath, **_kwargs): return f"uploaded:{Path(filepath).name}" -def _invoke(runner, tmp_path, extra_args, mocks): - package_file = _touch(tmp_path, "pkg_1.0-1_amd64.deb") +def _invoke(runner, tmp_path, extra_args, mocks, package_file=None): + if package_file is None: + package_file = _touch(tmp_path, "pkg_1.0-1_amd64.deb") with ( patch(_MOCK_TARGETS[0]) as mock_validate_create_package, patch(_MOCK_TARGETS[1], return_value="checksum") as mock_validate_upload, @@ -145,6 +146,56 @@ def test_dsc_maps_real_source_package_members_to_uploaded_sdk_fields( ) +def test_dsc_as_package_file_is_parsed_without_the_option(runner, tmp_path): + source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") + debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") + dsc_path = _write_dsc( + tmp_path, + [source_archive.name, debian_archive.name], + version="1.0-1", + source_format="3.0 (quilt)", + ) + + mocks = {} + result = _invoke(runner, tmp_path, [], mocks, package_file=dsc_path) + + assert result.exit_code == 0, result.output + kwargs = mocks["validate_create_package"].call_args.kwargs + assert kwargs["sources_file"] == str(source_archive) + assert kwargs["changes_file"] == str(debian_archive) + + +def test_explicit_sources_file_stops_the_package_file_being_parsed(runner, tmp_path): + # The .dsc references a member that is absent, so parsing it at all would + # abort the push; --sources-file means the caller drives the members. + explicit_source = _touch(tmp_path, "explicit.tar.gz") + dsc_path = _write_dsc(tmp_path, ["absent_1.0.tar.gz"]) + + mocks = {} + result = _invoke( + runner, + tmp_path, + ["--sources-file", str(explicit_source)], + mocks, + package_file=dsc_path, + ) + + assert result.exit_code == 0, result.output + kwargs = mocks["validate_create_package"].call_args.kwargs + assert kwargs["sources_file"] == str(explicit_source) + assert kwargs["changes_file"] is None + + +def test_binary_package_file_is_never_parsed_as_a_dsc(runner, tmp_path): + mocks = {} + result = _invoke(runner, tmp_path, [], mocks) + + assert result.exit_code == 0, result.output + kwargs = mocks["validate_create_package"].call_args.kwargs + assert kwargs["sources_file"] is None + assert kwargs["changes_file"] is None + + def test_clearsigned_dsc_is_parsed_through_registered_command(runner, tmp_path): source_archive = _touch(tmp_path, "pkg_1.0.orig.tar.gz") debian_archive = _touch(tmp_path, "pkg_1.0-1.debian.tar.xz") @@ -238,7 +289,7 @@ def test_member_path_is_rejected_before_network_calls(runner, tmp_path, member): mocks["validate_upload_file"].assert_not_called() -def test_escaping_symlink_is_rejected_before_network_calls(runner, tmp_path): +def test_symlinked_member_uploads_its_canonical_target(runner, tmp_path): dsc_dir = tmp_path / "dsc" dsc_dir.mkdir() outside_archive = _touch(tmp_path, "outside.tar.gz") @@ -248,8 +299,24 @@ def test_escaping_symlink_is_rejected_before_network_calls(runner, tmp_path): mocks = {} result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + assert result.exit_code == 0, result.output + _, kwargs = mocks["create_package"].call_args + assert kwargs["sources_file"] == "uploaded:outside.tar.gz" + + +def test_member_symlinked_to_a_directory_is_rejected_before_network_calls( + runner, tmp_path +): + dsc_dir = tmp_path / "dsc" + dsc_dir.mkdir() + (dsc_dir / "pkg_1.0.tar.gz").symlink_to(tmp_path, target_is_directory=True) + dsc_path = _write_dsc(dsc_dir, ["pkg_1.0.tar.gz"]) + + mocks = {} + result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) + assert result.exit_code != 0 - assert "not a regular file directly next to the .dsc" in result.output + assert "not a regular file next to the .dsc" in result.output mocks["validate_create_package"].assert_not_called() mocks["validate_upload_file"].assert_not_called() @@ -272,7 +339,7 @@ def test_multi_component_is_rejected_before_network_calls(runner, tmp_path): mocks["validate_create_package"].assert_not_called() -def test_detached_signature_is_rejected_before_network_calls(runner, tmp_path): +def test_detached_signature_is_skipped_with_a_warning(runner, tmp_path): filenames = [ _touch(tmp_path, "pkg_1.0.orig.tar.gz").name, _touch(tmp_path, "pkg_1.0.orig.tar.gz.asc").name, @@ -285,9 +352,11 @@ def test_detached_signature_is_rejected_before_network_calls(runner, tmp_path): mocks = {} result = _invoke(runner, tmp_path, ["--dsc-file", str(dsc_path)], mocks) - assert result.exit_code != 0 - assert "detached upstream signature" in result.output - mocks["validate_create_package"].assert_not_called() + assert result.exit_code == 0, result.output + assert "Not uploading pkg_1.0.orig.tar.gz.asc" in result.output + _, kwargs = mocks["create_package"].call_args + assert kwargs["sources_file"] == "uploaded:pkg_1.0.orig.tar.gz" + assert kwargs["changes_file"] == "uploaded:pkg_1.0-1.debian.tar.xz" def test_push_deb_help_documents_dsc_file_option(runner): From 7497ad2b71edbcd2c9535e4441e7bdd7c63c3a34 Mon Sep 17 00:00:00 2001 From: BB <55028730+BartoszBlizniak@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:09:09 +0100 Subject: [PATCH 4/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f17ec28..eddc6bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - `cloudsmith push deb` now derives a Debian source package's members from its `.dsc`, so `cloudsmith push deb /// foo_1.0-1.dsc` is enough where `--sources-file` and `--changes-file` previously had to be worked out by hand (and their suffixes vary: `.orig.tar.gz`, `.orig.tar.bz2`, `.debian.tar.xz`, `.diff.gz`, ...). The `Checksums-Sha256:` or `Files:` field of the `.dsc` is read — plain or OpenPGP-clearsigned — and the upstream/native source archive becomes `--sources-file` while the Debian packaging archive becomes `--changes-file`, for the `1.0`, `2.0`, `3.0 (native)` and `3.0 (quilt)` source formats. `--dsc-file` names a `.dsc` other than `PACKAGE_FILE`, and an explicit `--sources-file` or `--changes-file` still wins for its own field. A detached upstream signature (`*.orig.tar.*.asc`) is skipped with a warning, since the deb package format has no field to carry it; a multi-component source package (`*.orig-.tar.*`) is rejected outright, because leaving a component behind would upload incomplete source. - -### Added - - `cloudsmith domains list` lists the hosts Cloudsmith can authenticate as a versioned JSON document: `{"version": 1, "domains": [{"host": ..., "format": ..., "type": ..., "domain_type": ..., "org": ..., "repository": ..., "primary": ..., "created_at": ...}]}`. The built-in list can be replaced by a `[domains]` section in a trusted `config.ini` — each entry maps a hostname to the format it serves, or to `download`/`upload` — for dedicated deployments. An organisation's own custom domains are listed ahead of the built-in hosts, and a custom domain that is disabled or not yet validated is left out entirely, since it serves nothing. `--format` and `--repo` narrow the list to the hosts usable for a package format or repository, most-preferred first, and `--domain-type` to those with one purpose: `download`, `upload`, `api` or `native_api`. - The Cloudsmith organisation is now named by `--org`, with `--organization` and `--oidc-org` accepted as aliases for the same option, and `org`, `organization` or `oidc_org` accepted in `config.ini`. `--oidc-org` named the setting after the first feature that wanted it; it is read by custom-domain discovery as well as OIDC token exchange, so it is now named after what it is. The `CLOUDSMITH_ORG` environment variable is unchanged, and `credential-helper install` no longer has a separate `--org` of its own.