-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathextraction_utils.py
More file actions
2574 lines (2310 loc) · 93.6 KB
/
Copy pathextraction_utils.py
File metadata and controls
2574 lines (2310 loc) · 93.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
from __future__ import annotations
import errno
import io
import os
import re
import shutil
import stat
import subprocess
import tempfile
import threading
import time
import uuid
import zipfile
from dataclasses import dataclass, field
from enum import Enum
from pathlib import PurePosixPath
from typing import Any, Callable, Iterable, Sequence
from PySide6.QtCore import QThread, Signal
from logger_utils import get_logger
from utils import (
WINDOWS_RESERVED_NAMES,
downloads_dir,
find_7z_executable,
find_unrar_executable,
hidden_subprocess_kwargs,
)
log = get_logger(__name__)
ARCHIVE_SUFFIXES = (".zip", ".rar", ".7z")
MAX_ARCHIVE_ENTRIES = 200_000
MAX_TOTAL_UNCOMPRESSED_BYTES = 100 * 1024**3
MAX_FILE_UNCOMPRESSED_BYTES = 50 * 1024**3
MAX_EMBEDDED_ARCHIVES = 100
MAX_NESTING_DEPTH = 1
MIN_FREE_SPACE_BYTES = 2 * 1024**3
SUSPICIOUS_RATIO = 1000
SUSPICIOUS_RATIO_MIN_BYTES = 1024**3
EXTERNAL_TOTAL_TIMEOUT_SECONDS = 4 * 60 * 60
EXTERNAL_PROGRESS_TIMEOUT_SECONDS = 5 * 60
MAX_EXTERNAL_OUTPUT_CHARS = 256 * 1024**2
MAX_EXTERNAL_LINE_CHARS = 64 * 1024
MAX_EXTERNAL_RECORD_FIELDS = 64
COPY_CHUNK_SIZE = 1024 * 1024
_IGNORED_NAMES = {".ds_store", "thumbs.db", "desktop.ini", "__macosx"}
_INVALID_WINDOWS_CHARS = set('<>"|?*')
_TEMPLATE_WORD = re.compile(r"(?<![A-Za-z0-9])templates?(?![A-Za-z0-9])", re.IGNORECASE)
class ConflictPolicy(str, Enum):
REPLACE = "replace"
SKIP = "skip"
CANCEL = "cancel"
@classmethod
def coerce(cls, value: ConflictPolicy | str) -> ConflictPolicy:
if isinstance(value, cls):
return value
try:
return cls(str(value).casefold())
except ValueError as exc:
raise ValueError(f"Unknown conflict policy: {value}") from exc
@dataclass(frozen=True)
class ArchiveMember:
path: str
size: int
compressed_size: int
is_dir: bool = False
@dataclass(frozen=True)
class ArchiveInventory:
archive_path: str
members: tuple[ArchiveMember, ...]
total_uncompressed: int
embedded_archives: int
@dataclass(frozen=True)
class ExtractionBuildPlan:
part: int
archive_path: str
content_dir: str = ""
build_id: str | None = None
@dataclass(frozen=True)
class ExtractionPlan:
builds: tuple[ExtractionBuildPlan, ...]
template_archives: tuple[str, ...] = ()
conflict_policy: ConflictPolicy = ConflictPolicy.CANCEL
@dataclass(frozen=True)
class PlannedEmbeddedArchive:
relative_path: str
staged_path: str
@dataclass
class ArchiveImportPlan:
stage_root: str
direct_content_root: str | None
content_archives: tuple[PlannedEmbeddedArchive, ...] = ()
template_archives: tuple[PlannedEmbeddedArchive, ...] = ()
ignored_archives: tuple[PlannedEmbeddedArchive, ...] = ()
warning: str | None = None
budget_entries: int = 0
budget_uncompressed: int = 0
budget_embedded_archives: int = 0
_claimed: bool = field(default=False, init=False, repr=False)
_closed: bool = field(default=False, init=False, repr=False)
_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
@property
def is_direct_content(self) -> bool:
return self.direct_content_root is not None
def claim(self) -> None:
with self._lock:
if self._closed:
raise ExtractionError("Archive import plan has already been cleaned up.")
if self._claimed:
raise ExtractionError("Archive import plan has already been consumed.")
self._claimed = True
self.validate()
def validate(self) -> None:
stage_root = os.path.abspath(self.stage_root)
if not os.path.isdir(stage_root) or _is_link_or_reparse(stage_root):
raise ExtractionError("Archive import staging area is no longer available.")
paths = [item.staged_path for item in self.content_archives]
paths.extend(item.staged_path for item in self.template_archives)
paths.extend(item.staged_path for item in self.ignored_archives)
if self.direct_content_root:
paths.append(self.direct_content_root)
for path in paths:
candidate = os.path.abspath(path)
try:
contained = os.path.commonpath((stage_root, candidate)) == stage_root
except ValueError:
contained = False
if not contained or not os.path.exists(candidate) or _is_link_or_reparse(candidate):
raise UnsafeArchiveError("Archive import plan contains an unsafe staged path.")
def initial_budget(self):
return _ExtractionBudget(
self.budget_entries,
self.budget_uncompressed,
self.budget_embedded_archives,
)
def cleanup(self) -> None:
with self._lock:
if self._closed:
return
self._closed = True
shutil.rmtree(self.stage_root, ignore_errors=True)
@dataclass(frozen=True)
class ArchivePlanningResult:
status: str
message: str = ""
plan: ArchiveImportPlan | None = None
@property
def succeeded(self) -> bool:
return self.status == "success"
@dataclass
class ExtractionResult:
status: str
message: str = ""
modified_builds: list[str] = field(default_factory=list)
copied_templates: list[str] = field(default_factory=list)
skipped_files: list[str] = field(default_factory=list)
errors: list[str] = field(default_factory=list)
new_builds: list[dict[str, Any]] = field(default_factory=list)
next_build_number: int | None = None
_transaction: _FileTransaction | None = field(
default=None, repr=False, compare=False
)
@property
def succeeded(self) -> bool:
return self.status == "success"
@property
def cancelled(self) -> bool:
return self.status == "cancelled"
@property
def rollback_pending(self) -> bool:
return bool(
self._transaction is not None
and self._transaction.rollback_pending
)
def finalize(self) -> None:
"""Discard rollback backups after the UI persisted the session update."""
transaction = self._transaction
self._transaction = None
if transaction is not None:
transaction.finalize()
def rollback(self) -> None:
"""Restore committed files if applying or persisting the result fails."""
transaction = self._transaction
if transaction is not None:
transaction.rollback()
self._transaction = None
class ExtractionError(RuntimeError):
pass
class ExtractionRollbackError(ExtractionError):
def __init__(
self,
failures: Sequence[tuple[str, OSError]],
*,
original_error: BaseException | None = None,
):
self.failures = tuple(failures)
self.original_error = original_error
preview = "; ".join(
f"{path}: {error}" for path, error in self.failures[:3]
)
if len(self.failures) > 3:
preview += f"; and {len(self.failures) - 3} more"
super().__init__(
f"Rollback incomplete for {len(self.failures)} path(s): {preview}"
)
class UnsafeArchiveError(ExtractionError):
pass
class ArchiveToolUnavailable(ExtractionError):
pass
class ExtractionCancelled(ExtractionError):
pass
class ExtractionConflict(ExtractionCancelled):
pass
class MultipartArchiveError(ValueError):
pass
def _check_cancelled(cancel_check: Callable[[], bool] | None) -> None:
if cancel_check is not None and cancel_check():
raise ExtractionCancelled("Extraction cancelled.")
def is_template_archive(path: str) -> bool:
stem = os.path.splitext(os.path.basename(path))[0]
return bool(_TEMPLATE_WORD.search(stem))
def classify_archives(archive_files, enable_template_detection):
content_archives = []
template_archives = []
ignored_archives = []
for archive_path in archive_files:
if is_template_archive(archive_path):
if enable_template_detection:
template_archives.append(archive_path)
else:
ignored_archives.append(archive_path)
else:
content_archives.append(archive_path)
return content_archives, template_archives, ignored_archives
def _ordered_parts(
matches: Sequence[tuple[int, int | None, str]],
pattern_name: str,
) -> list[str]:
part_numbers = [part for part, _, _ in matches]
if len(part_numbers) != len(set(part_numbers)):
raise MultipartArchiveError(
f"Duplicate part numbers detected in {pattern_name}: {part_numbers}"
)
declared_totals = {total for _, total, _ in matches if total is not None}
if len(declared_totals) > 1:
raise MultipartArchiveError(
f"Conflicting multipart totals detected in {pattern_name}."
)
expected_total = next(iter(declared_totals), len(matches))
if expected_total < 1 or expected_total > 99:
raise MultipartArchiveError(
f"Multipart total must be between 1 and 99, got {expected_total}."
)
expected = set(range(1, expected_total + 1))
actual = set(part_numbers)
if actual != expected:
missing = sorted(expected - actual)
extra = sorted(actual - expected)
details = []
if missing:
details.append(f"missing parts {missing}")
if extra:
details.append(f"unexpected parts {extra}")
raise MultipartArchiveError(
f"Incomplete multipart archive ({', '.join(details)})."
)
return [path for _, _, path in sorted(matches, key=lambda item: item[0])]
def detect_heuristic_ordering(archive_files):
"""Order multipart archives and reject duplicate or incomplete sequences."""
archive_files = list(archive_files)
if not archive_files:
return [], None
if len(archive_files) > 99:
raise MultipartArchiveError("A DIM package can contain at most 99 parts.")
patterns = (
(
"XofY pattern",
re.compile(r"_(\d+)of(\d+)(?=\D|$)", re.IGNORECASE),
True,
),
(
"Part pattern",
re.compile(
r"(?:^|[^A-Za-z0-9])part\s*(\d+)"
r"(?:\s*(?:of|-)\s*(\d+))?(?=\D|$)",
re.IGNORECASE,
),
True,
),
(
"build-number pattern",
re.compile(r"(?<!\d)(\d{1,2})_\d{5,}\.(?:zip|rar|7z)$", re.IGNORECASE),
False,
),
(
"trailing-number pattern",
re.compile(r"(?<!\d)_(\d{1,2})\.(?:zip|rar|7z)$", re.IGNORECASE),
False,
),
)
for pattern_name, pattern, supports_total in patterns:
matches = []
for archive_path in archive_files:
match = pattern.search(os.path.basename(archive_path))
if not match:
continue
total = int(match.group(2)) if supports_total and match.lastindex and match.lastindex >= 2 and match.group(2) else None
matches.append((int(match.group(1)), total, archive_path))
if matches:
if len(matches) != len(archive_files):
raise MultipartArchiveError(
f"Mixed or incomplete {pattern_name}: every selected archive "
"must use the same numbering scheme."
)
return _ordered_parts(matches, pattern_name), None
if len(archive_files) == 1:
return archive_files, None
return (
sorted(archive_files, key=lambda path: os.path.basename(path).casefold()),
"Could not detect build numbering pattern. Archives ordered alphabetically.",
)
def _normalise_member_path(raw_path: str) -> str:
if not isinstance(raw_path, str) or not raw_path or "\x00" in raw_path:
raise UnsafeArchiveError("Archive contains an empty or invalid path.")
path = raw_path.replace("\\", "/")
if path.startswith("/") or path.startswith("//") or re.match(r"^[A-Za-z]:", path):
raise UnsafeArchiveError(f"Archive contains an absolute path: {raw_path}")
parts = []
for part in PurePosixPath(path).parts:
if part in ("", "."):
continue
if part == "..":
raise UnsafeArchiveError(f"Archive path escapes its destination: {raw_path}")
if part.endswith((" ", ".")):
raise UnsafeArchiveError(f"Archive path has a trailing dot or space: {raw_path}")
if ":" in part:
raise UnsafeArchiveError(f"Archive path contains an NTFS stream or drive: {raw_path}")
if any(char in _INVALID_WINDOWS_CHARS or ord(char) < 32 for char in part):
raise UnsafeArchiveError(f"Archive path contains invalid Windows characters: {raw_path}")
device_name = part.split(".", 1)[0].rstrip(" .").upper()
if device_name in WINDOWS_RESERVED_NAMES:
raise UnsafeArchiveError(f"Archive path uses a Windows device name: {raw_path}")
parts.append(part)
if not parts:
raise UnsafeArchiveError(f"Archive contains an invalid root entry: {raw_path}")
return "/".join(parts)
def _is_archive_name(path: str) -> bool:
return path.casefold().endswith(ARCHIVE_SUFFIXES)
def _validate_inventory(
archive_path: str,
members: Iterable[ArchiveMember],
cancel_check: Callable[[], bool] | None = None,
) -> ArchiveInventory:
validated = []
explicit_members: dict[str, str] = {}
path_nodes: dict[str, tuple[str, bool]] = {}
total_size = 0
total_file_size = 0
total_compressed = 0
embedded_count = 0
for member in members:
_check_cancelled(cancel_check)
path = _normalise_member_path(member.path)
key = path.casefold()
previous = explicit_members.get(key)
if previous is not None:
raise UnsafeArchiveError(
f"Archive contains a case-insensitive path collision: {previous} / {path}"
)
explicit_members[key] = path
parts = path.split("/")
for index in range(1, len(parts) + 1):
node_path = "/".join(parts[:index])
node_key = node_path.casefold()
node_is_dir = index < len(parts) or member.is_dir
existing = path_nodes.get(node_key)
if existing is None:
path_nodes[node_key] = (node_path, node_is_dir)
continue
existing_path, existing_is_dir = existing
if existing_path != node_path:
raise UnsafeArchiveError(
"Archive contains a case-insensitive path collision: "
f"{existing_path} / {node_path}"
)
if existing_is_dir != node_is_dir:
raise UnsafeArchiveError(
"Archive uses a file as a parent directory or reuses a path "
f"as both file and directory: {node_path}"
)
if member.size < 0 or member.compressed_size < 0:
raise UnsafeArchiveError(f"Archive contains an invalid size for {path}.")
if member.size > MAX_FILE_UNCOMPRESSED_BYTES:
raise UnsafeArchiveError(
f"Archive member exceeds the 50 GiB limit: {path}"
)
if (
not member.is_dir
and member.size >= SUSPICIOUS_RATIO_MIN_BYTES
and member.size / max(member.compressed_size, 1) > SUSPICIOUS_RATIO
):
raise UnsafeArchiveError(
f"Archive member has a suspicious compression ratio: {path}"
)
total_size += member.size
if not member.is_dir:
total_file_size += member.size
total_compressed += member.compressed_size
if total_size > MAX_TOTAL_UNCOMPRESSED_BYTES:
raise UnsafeArchiveError("Archive exceeds the 100 GiB uncompressed size limit.")
if not member.is_dir and _is_archive_name(path):
embedded_count += 1
validated.append(
ArchiveMember(path, member.size, member.compressed_size, member.is_dir)
)
if len(validated) > MAX_ARCHIVE_ENTRIES:
raise UnsafeArchiveError("Archive exceeds the 200,000 entry limit.")
if embedded_count > MAX_EMBEDDED_ARCHIVES:
raise UnsafeArchiveError("Archive contains more than 100 embedded archives.")
if (
total_file_size >= SUSPICIOUS_RATIO_MIN_BYTES
and total_file_size / max(total_compressed, 1) > SUSPICIOUS_RATIO
):
raise UnsafeArchiveError(
"Archive has a suspicious aggregate compression ratio."
)
return ArchiveInventory(
archive_path=os.path.abspath(archive_path),
members=tuple(validated),
total_uncompressed=total_size,
embedded_archives=embedded_count,
)
def _stat_is_link_or_reparse(info: os.stat_result) -> bool:
if stat.S_ISLNK(info.st_mode):
return True
attributes = getattr(info, "st_file_attributes", 0)
reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
return bool(attributes & reparse_flag)
def _is_link_or_reparse(path: str) -> bool:
try:
info = os.lstat(path)
except OSError:
return False
return _stat_is_link_or_reparse(info)
def _safe_destination(root: str, relative_path: str) -> str:
relative_path = _normalise_member_path(relative_path)
root_abs = os.path.abspath(root)
destination = os.path.abspath(
os.path.join(root_abs, *relative_path.split("/"))
)
try:
contained = os.path.commonpath((root_abs, destination)) == root_abs
except ValueError:
contained = False
if not contained:
raise UnsafeArchiveError(f"Path escapes extraction root: {relative_path}")
return destination
def _ensure_free_space(path: str, required_bytes: int) -> None:
probe = os.path.abspath(path)
while not os.path.exists(probe):
parent = os.path.dirname(probe)
if parent == probe:
break
probe = parent
free = shutil.disk_usage(probe).free
if free - required_bytes < MIN_FREE_SPACE_BYTES:
raise ExtractionError(
"Not enough free disk space. At least 2 GiB must remain after extraction."
)
def _volume_key(path: str) -> tuple[str, int | str]:
"""Return a stable key for aggregating planned writes on one volume."""
probe = os.path.abspath(path)
while not os.path.exists(probe):
parent = os.path.dirname(probe)
if parent == probe:
break
probe = parent
try:
device = os.stat(probe).st_dev
except OSError:
device = 0
drive = os.path.splitdrive(probe)[0].casefold()
return ("device", device) if device else ("drive", drive or probe.casefold())
def _copy_file(
source: str,
destination: str,
cancel_check: Callable[[], bool],
*,
expected_stat: os.stat_result | None = None,
) -> None:
def identity(info: os.stat_result) -> tuple[int, int, int, int]:
return (
info.st_dev,
info.st_ino,
info.st_size,
info.st_mtime_ns,
)
source = os.path.abspath(source)
destination = os.path.abspath(destination)
source_before = os.lstat(source)
if expected_stat is not None and identity(source_before) != identity(expected_stat):
raise UnsafeArchiveError(
"Source file changed before it could be copied."
)
if (
not stat.S_ISREG(source_before.st_mode)
or _stat_is_link_or_reparse(source_before)
):
raise UnsafeArchiveError("Copy source must be a regular file.")
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0)
flags |= getattr(os, "O_NOFOLLOW", 0)
source_fd = -1
destination_created = False
final_source_stat = source_before
try:
try:
source_fd = os.open(source, flags)
except OSError as exc:
if exc.errno == errno.ELOOP:
raise UnsafeArchiveError(
"Copy source became a link while being opened."
) from exc
raise
opened_stat = os.fstat(source_fd)
if (
not stat.S_ISREG(opened_stat.st_mode)
or _stat_is_link_or_reparse(opened_stat)
):
raise UnsafeArchiveError("Copy source must be a regular file.")
if identity(opened_stat) != identity(source_before):
raise UnsafeArchiveError(
"Source file changed while being opened."
)
os.makedirs(os.path.dirname(destination), exist_ok=True)
copied_bytes = 0
with os.fdopen(source_fd, "rb") as src:
source_fd = -1
with open(destination, "xb") as dst:
destination_created = True
while True:
if cancel_check():
raise ExtractionCancelled("Extraction cancelled.")
chunk = src.read(COPY_CHUNK_SIZE)
if not chunk:
break
dst.write(chunk)
copied_bytes += len(chunk)
final_source_stat = os.fstat(src.fileno())
if (
identity(final_source_stat) != identity(source_before)
or copied_bytes != source_before.st_size
):
raise UnsafeArchiveError(
"Source file changed while it was being copied."
)
except BaseException:
if destination_created and os.path.lexists(destination):
try:
os.remove(destination)
except OSError as cleanup_error:
log.warning(
"Failed to remove incomplete copied file %s: %s",
destination,
cleanup_error,
)
raise
finally:
if source_fd >= 0:
os.close(source_fd)
try:
os.chmod(destination, stat.S_IMODE(final_source_stat.st_mode))
os.utime(
destination,
ns=(final_source_stat.st_atime_ns, final_source_stat.st_mtime_ns),
)
except OSError:
pass
def _snapshot_archive(
source_path: str,
snapshot_root: str,
cancel_check: Callable[[], bool],
) -> str:
"""Copy an untrusted archive so inventory and extraction share exact bytes."""
source = os.path.abspath(source_path)
try:
source_stat = os.lstat(source)
except OSError as exc:
raise ExtractionError(f"Cannot read archive: {source}") from exc
if not stat.S_ISREG(source_stat.st_mode) or _is_link_or_reparse(source):
raise UnsafeArchiveError("Archive source must be a regular file.")
suffix = os.path.splitext(source)[1].casefold()
if suffix not in ARCHIVE_SUFFIXES:
raise ExtractionError(f"Unsupported archive type: {suffix or '<none>'}")
_ensure_free_space(snapshot_root, source_stat.st_size)
os.makedirs(snapshot_root, exist_ok=True)
snapshot = os.path.join(snapshot_root, f"source{suffix}")
_copy_file(
source,
snapshot,
cancel_check,
expected_stat=source_stat,
)
return snapshot
def _stop_process(process: subprocess.Popen) -> None:
if process.poll() is not None:
return
try:
process.terminate()
process.wait(timeout=3)
except Exception:
try:
process.kill()
process.wait(timeout=3)
except Exception:
pass
def _run_external(
arguments: Sequence[str],
cancel_check: Callable[[], bool],
*,
output_parser: Callable[[Iterable[str], Callable[[], bool]], Any] | None = None,
capture_output: bool = True,
) -> Any:
"""Run an archive tool while spooling merged output outside process memory."""
with tempfile.TemporaryFile(mode="w+b", buffering=0) as spool:
process = subprocess.Popen(
list(arguments),
shell=False,
stdin=subprocess.DEVNULL,
stdout=spool,
stderr=subprocess.STDOUT,
**hidden_subprocess_kwargs(),
)
started = time.monotonic()
last_progress = started
observed_length = 0
try:
while process.poll() is None:
if cancel_check():
raise ExtractionCancelled("Extraction cancelled.")
now = time.monotonic()
if now - started > EXTERNAL_TOTAL_TIMEOUT_SECONDS:
raise ExtractionError(
"Archive tool exceeded the four-hour time limit."
)
output_length = os.fstat(spool.fileno()).st_size
if output_length > MAX_EXTERNAL_OUTPUT_CHARS:
raise UnsafeArchiveError(
"Archive tool produced excessive output."
)
if output_length > observed_length:
observed_length = output_length
last_progress = now
elif now - last_progress > EXTERNAL_PROGRESS_TIMEOUT_SECONDS:
raise ExtractionError(
"Archive tool made no progress for five minutes."
)
time.sleep(0.1)
output_length = os.fstat(spool.fileno()).st_size
if output_length > MAX_EXTERNAL_OUTPUT_CHARS:
raise UnsafeArchiveError("Archive tool produced excessive output.")
if process.returncode != 0:
spool.seek(max(0, output_length - 4096))
detail = spool.read().decode("utf-8", errors="replace")
lowered = detail.casefold()
if "password" in lowered or "encrypted" in lowered:
raise UnsafeArchiveError(
"Password-protected archives are not supported."
)
raise ExtractionError(
"Archive tool failed with exit code "
f"{process.returncode}: {detail.strip()[-1200:]}"
)
if output_parser is not None:
spool.seek(0)
text_output = io.TextIOWrapper(
spool, encoding="utf-8", errors="replace", newline=None
)
try:
return output_parser(text_output, cancel_check)
finally:
text_output.detach()
if not capture_output:
return ""
spool.seek(0)
return spool.read().decode("utf-8", errors="replace")
except BaseException:
_stop_process(process)
raise
class _ZipAdapter:
def __init__(
self,
archive_path: str,
cancel_check: Callable[[], bool] | None = None,
):
self.archive_path = archive_path
self.cancel_check = cancel_check
try:
self._zip = zipfile.ZipFile(archive_path, "r")
except (OSError, zipfile.BadZipFile) as exc:
raise ExtractionError(f"Invalid ZIP archive: {os.path.basename(archive_path)}") from exc
self._infos: list[zipfile.ZipInfo] = []
def __enter__(self):
return self
def __exit__(self, *_):
self._zip.close()
def inventory(self) -> ArchiveInventory:
members = []
infos = self._zip.infolist()
if len(infos) > MAX_ARCHIVE_ENTRIES:
raise UnsafeArchiveError("Archive exceeds the 200,000 entry limit.")
for info in infos:
_check_cancelled(self.cancel_check)
if info.flag_bits & 0x1:
raise UnsafeArchiveError("Password-protected archives are not supported.")
unix_mode = info.external_attr >> 16
file_type = stat.S_IFMT(unix_mode)
if file_type == stat.S_IFLNK:
raise UnsafeArchiveError(f"Archive contains a symbolic link: {info.filename}")
if file_type not in (0, stat.S_IFREG, stat.S_IFDIR):
raise UnsafeArchiveError(f"Archive contains a special file: {info.filename}")
dos_attributes = info.external_attr & 0xFFFF
if dos_attributes & 0x400:
raise UnsafeArchiveError(f"Archive contains a reparse point: {info.filename}")
members.append(
ArchiveMember(
info.filename,
info.file_size,
info.compress_size,
info.is_dir(),
)
)
inventory = _validate_inventory(
self.archive_path, members, self.cancel_check
)
self._infos = infos
return inventory
def extract(
self,
destination: str,
inventory: ArchiveInventory,
cancel_check: Callable[[], bool],
) -> None:
if not self._infos:
raise ExtractionError("ZIP archive was not inventoried before extraction.")
os.makedirs(destination, exist_ok=True)
inventory_by_key = {member.path.casefold(): member for member in inventory.members}
for info in self._infos:
if cancel_check():
raise ExtractionCancelled("Extraction cancelled.")
normalised = _normalise_member_path(info.filename)
member = inventory_by_key[normalised.casefold()]
target = _safe_destination(destination, normalised)
if member.is_dir:
os.makedirs(target, exist_ok=True)
continue
os.makedirs(os.path.dirname(target), exist_ok=True)
if os.path.lexists(target):
raise UnsafeArchiveError(f"Archive member would overwrite another member: {normalised}")
copied = 0
try:
with self._zip.open(info, "r") as source, open(target, "xb") as output:
while True:
if cancel_check():
raise ExtractionCancelled("Extraction cancelled.")
chunk = source.read(COPY_CHUNK_SIZE)
if not chunk:
break
output.write(chunk)
copied += len(chunk)
except ExtractionCancelled:
raise
except (OSError, zipfile.BadZipFile, RuntimeError) as exc:
raise ExtractionError(f"Failed to extract ZIP member {normalised}: {exc}") from exc
if copied != member.size:
raise ExtractionError(f"ZIP member size changed during extraction: {normalised}")
_validate_extracted_tree(destination, inventory, cancel_check)
def _iter_external_lines(output: str | Iterable[str]) -> Iterable[str]:
if isinstance(output, str):
for line in output.splitlines():
if len(line) > MAX_EXTERNAL_LINE_CHARS:
raise UnsafeArchiveError("Archive inventory contains an excessive line.")
yield line
return
readline = getattr(output, "readline", None)
if callable(readline):
while True:
line = readline(MAX_EXTERNAL_LINE_CHARS + 1)
if not line:
return
if len(line) > MAX_EXTERNAL_LINE_CHARS:
raise UnsafeArchiveError(
"Archive inventory contains an excessive line."
)
yield line
reader = iter(output)
while True:
try:
line = next(reader)
except StopIteration:
return
if len(line) > MAX_EXTERNAL_LINE_CHARS:
raise UnsafeArchiveError("Archive inventory contains an excessive line.")
yield line
def _append_external_record(
records: list[dict[str, str]], current: dict[str, str]
) -> None:
if len(records) >= MAX_ARCHIVE_ENTRIES + 16:
raise UnsafeArchiveError("Archive exceeds the 200,000 entry limit.")
records.append(current)
def _set_external_record_value(
current: dict[str, str], key: str, value: str
) -> None:
if key not in current and len(current) >= MAX_EXTERNAL_RECORD_FIELDS:
raise UnsafeArchiveError("Archive inventory record has excessive metadata.")
current[key] = value
def _parse_slt_records(
output: str | Iterable[str],
cancel_check: Callable[[], bool] | None = None,
) -> list[dict[str, str]]:
records = []
current = {}
for line in _iter_external_lines(output):
_check_cancelled(cancel_check)
if " = " not in line:
if not line.strip() and current:
_append_external_record(records, current)
current = {}
continue
key, value = line.split(" = ", 1)
if key == "Path" and "Path" in current:
_append_external_record(records, current)
current = {}
_set_external_record_value(current, key.strip(), value.strip())
if current:
_append_external_record(records, current)
return records
def _parse_colon_records(
output: str | Iterable[str],
cancel_check: Callable[[], bool] | None = None,
) -> list[dict[str, str]]:
records = []
current = {}
for line in _iter_external_lines(output):
_check_cancelled(cancel_check)
stripped = line.strip()
if not stripped:
if current:
_append_external_record(records, current)
current = {}
continue
if ":" not in stripped:
continue
key, value = stripped.split(":", 1)
if key.casefold() == "name" and "Name" in current:
_append_external_record(records, current)
current = {}
_set_external_record_value(
current, key.strip().title(), value.strip()
)
if current:
_append_external_record(records, current)
return records
def _integer(record: dict[str, str], *keys: str) -> int:
for key in keys:
value = record.get(key)
if value is not None:
digits = value.replace(",", "").replace(" ", "")
if digits.isdigit():
return int(digits)