diff --git a/.gitignore b/.gitignore index 8d73cb7..e1c0e17 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ target # keep your zip files! **/*.zip + +packobf_gui/.qmlls.ini diff --git a/java/rust/src/lib.rs b/java/rust/src/lib.rs index 60625b9..1abd0df 100644 --- a/java/rust/src/lib.rs +++ b/java/rust/src/lib.rs @@ -30,7 +30,7 @@ pub extern "system" fn Java_dev_misieur_packobf_Native_optimizeZip<'caller>( }; let options = if options.is_null() { - Options::simplest() + Options::fastest() } else { let comp_val = env .get_field(&options, jni_str!("compression"), jni_sig!("I"))? @@ -41,9 +41,10 @@ pub extern "system" fn Java_dev_misieur_packobf_Native_optimizeZip<'caller>( Options { compression: match comp_val { - 0 => Compression::Simplest, - 1 => Compression::Normal, - _ => Compression::Max, + 0 => Compression::Fastest, + 1 => Compression::Fast, + 2 => Compression::Normal, + _ => Compression::Best, }, shader_compression: match shader_comp_val { 0 => ShaderCompression::None, @@ -59,6 +60,10 @@ pub extern "system" fn Java_dev_misieur_packobf_Native_optimizeZip<'caller>( corrupt_png_files: env .get_field(&options, jni_str!("corruptPngFiles"), jni_sig!("Z"))? .z()?, + num_threads: Some( + env.get_field(&options, jni_str!("numThreads"), jni_sig!("I"))? + .i()? as usize, + ), } }; @@ -119,12 +124,17 @@ pub extern "system" fn Java_dev_misieur_packobf_Native_optimizeZip<'caller>( total: t, } => (1, c as i32, t as i32, None), Progress::Parsing { current: s } => (2, 0, 0, Some(s)), - Progress::Building { + Progress::Optimizing { current: s, index: i, total: t, } => (3, i as i32, t as i32, Some(s)), - Progress::Done => (4, 0, 0, None), + Progress::Building { + current: s, + index: i, + total: t, + } => (4, i as i32, t as i32, Some(s)), + Progress::Done => (5, 0, 0, None), }; let _ = env.with_local_frame(16, |env| { diff --git a/java/src/main/java/dev/misieur/packobf/Native.java b/java/src/main/java/dev/misieur/packobf/Native.java index adc421b..2d8479a 100644 --- a/java/src/main/java/dev/misieur/packobf/Native.java +++ b/java/src/main/java/dev/misieur/packobf/Native.java @@ -14,12 +14,13 @@ private Native() { static class Options { - public Options(int compression, int shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles) { + public Options(int compression, int shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles, int numThreads) { this.compression = compression; this.shaderCompression = shaderCompression; this.renameFiles = renameFiles; this.blockUnzipping = blockUnzipping; this.corruptPngFiles = corruptPngFiles; + this.numThreads = numThreads; } public int compression; @@ -27,6 +28,7 @@ public Options(int compression, int shaderCompression, boolean renameFiles, bool public boolean renameFiles; public boolean blockUnzipping; public boolean corruptPngFiles; + public int numThreads; } interface LogCallback { diff --git a/java/src/main/java/dev/misieur/packobf/PackOBF.java b/java/src/main/java/dev/misieur/packobf/PackOBF.java index 1aebf9b..cee4a19 100644 --- a/java/src/main/java/dev/misieur/packobf/PackOBF.java +++ b/java/src/main/java/dev/misieur/packobf/PackOBF.java @@ -42,7 +42,8 @@ public static byte[] optimizeZip( options.shaderCompression().value, options.renameFiles(), options.blockUnzipping(), - options.corruptPngFiles() + options.corruptPngFiles(), + options.numThreads() != null ? options.numThreads() : 0 ), (level, message) -> logCallback.onLog(switch (level) { case 0 -> LogLevel.INFO; diff --git a/java/src/main/java/dev/misieur/packobf/options/Options.java b/java/src/main/java/dev/misieur/packobf/options/Options.java index e01c400..8d7ddf1 100644 --- a/java/src/main/java/dev/misieur/packobf/options/Options.java +++ b/java/src/main/java/dev/misieur/packobf/options/Options.java @@ -1,13 +1,16 @@ package dev.misieur.packobf.options; -public record Options(Compression compression, ShaderCompression shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles) { +import dev.misieur.packobf.annotations.Nullable; + +public record Options(Compression compression, ShaderCompression shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles, @Nullable Integer numThreads) { public static Options simplest() { return new Options( Compression.SIMPLEST, ShaderCompression.NONE, false, false, - false + false, + null ); } @@ -17,7 +20,8 @@ public static Options normal() { ShaderCompression.NONE, false, false, - false + false, + null ); } @@ -28,7 +32,8 @@ public static Options max() { ShaderCompression.MINIFY_AND_OBFUSCATE, true, true, - true + true, + null ); } } diff --git a/java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java b/java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java new file mode 100644 index 0000000..224c5d2 --- /dev/null +++ b/java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java @@ -0,0 +1,33 @@ +package dev.misieur.packobf.progress; + +public final class OptimizingProgress extends Progress { + /** + * The total number of files to optimize + */ + private final int total; + /** + * The current file that PackOBF started to optimize + */ + private final Current current; + + public OptimizingProgress(int total, Current current) { + this.total = total; + this.current = current; + } + + public int total() { + return total; + } + + public Current current() { + return current; + } + + @Override + public State state() { + return State.OPTIMIZING; + } + + public record Current(String name, int index) { + } +} diff --git a/java/src/main/java/dev/misieur/packobf/progress/Progress.java b/java/src/main/java/dev/misieur/packobf/progress/Progress.java index 369fbeb..a394751 100644 --- a/java/src/main/java/dev/misieur/packobf/progress/Progress.java +++ b/java/src/main/java/dev/misieur/packobf/progress/Progress.java @@ -1,6 +1,6 @@ package dev.misieur.packobf.progress; -public abstract sealed class Progress permits BuildingProgress, DoneProgress, IdleProgress, ParsingProgress, ReadingZipProgress { +public abstract sealed class Progress permits IdleProgress, ReadingZipProgress, ParsingProgress, OptimizingProgress, BuildingProgress, DoneProgress { private State state; public abstract State state(); @@ -9,8 +9,9 @@ public enum State { IDLE(0), READING_ZIP(1), PARSING(2), - BUILDING(3), - DONE(4); + OPTIMIZING(3), + BUILDING(4), + DONE(5); private final int value; diff --git a/packobf/src/cache.rs b/packobf/src/cache.rs index 5192f57..c9935b1 100644 --- a/packobf/src/cache.rs +++ b/packobf/src/cache.rs @@ -6,12 +6,10 @@ use std::io::{BufReader, BufWriter, Read, Write}; use crate::profile_scope; -const MAGIC_NUMBER: [u8; 8] = *b"PACKOBF1"; -pub const VERSION: u16 = 1; +const MAGIC_NUMBER: [u8; 10] = *b"PACKOBF001"; // Increase version number each time compression is changed (hex number) pub struct CachedItem { pub compression: Compression, - pub version: u16, pub data: Vec, } @@ -19,16 +17,20 @@ pub struct CachedItem { #[derive(Debug, Clone, Copy)] pub enum Compression { Fastest = 0, - Normal = 1, - Best = 2, + Fast = 1, + Normal = 2, + Best = 3, + Ultra = 4, } impl Compression { fn from_u8(value: u8) -> Self { match value { 0 => Compression::Fastest, - 2 => Compression::Best, - _ => Compression::Normal, + 1 => Compression::Fast, + 2 => Compression::Normal, + 3 => Compression::Best, + _ => Compression::Ultra, } } } @@ -85,9 +87,6 @@ impl Cache { // Write Compression (1 byte) writer.write_all(&[item.compression as u8])?; - // Write Version (2 bytes) - writer.write_all(&item.version.to_le_bytes())?; - // Write Data Length (u64) and Data writer.write_all(&(item.data.len() as u64).to_le_bytes())?; writer.write_all(&item.data)?; @@ -101,25 +100,19 @@ impl Cache { profile_scope!("load_from_file::cache"); let file = File::open(path); if let Err(e) = file { - return if e.kind() == io::ErrorKind::NotFound { - Ok(Cache { - items: DashMap::new(), - }) - } else { - Err(e) - }; + return Err(e) } let file = file?; let mut reader = BufReader::new(file); let items = DashMap::new(); - let mut magic = [0u8; 8]; + let mut magic = [0u8; 10]; reader.read_exact(&mut magic)?; if magic != MAGIC_NUMBER { return Err(io::Error::new( io::ErrorKind::InvalidData, - "Not a valid cache file: Magic number mismatch", + "Not a valid cache file: Magic number mismatch (may be from a different version of packobf)", )); } @@ -143,11 +136,6 @@ impl Cache { reader.read_exact(&mut comp_byte)?; let compression = Compression::from_u8(comp_byte[0]); - // Read Version - let mut ver_bytes = [0u8; 2]; - reader.read_exact(&mut ver_bytes)?; - let version = u16::from_le_bytes(ver_bytes); - // Read Data Length and then the Data let mut data_len_bytes = [0u8; 8]; reader.read_exact(&mut data_len_bytes)?; @@ -156,16 +144,13 @@ impl Cache { let mut data = vec![0u8; data_len]; reader.read_exact(&mut data)?; - if version == VERSION { - items.insert( - CachedItemKey { hash, item_type }, - CachedItem { - compression, - version, - data, - }, - ); - } + items.insert( + CachedItemKey { hash, item_type }, + CachedItem { + compression, + data, + }, + ); } Ok(Cache { items }) @@ -198,7 +183,6 @@ impl Cache { CachedItemKey { hash, item_type }, CachedItem { compression: Compression::from_u8(compression), - version: VERSION, data: data.into(), }, ); @@ -218,7 +202,6 @@ impl Cache { }, CachedItem { compression: Compression::from_u8(compression), - version: VERSION, data: data.into(), }, ); diff --git a/packobf/src/file_parser.rs b/packobf/src/file_parser.rs index d1b6c19..1bfac68 100644 --- a/packobf/src/file_parser.rs +++ b/packobf/src/file_parser.rs @@ -16,6 +16,7 @@ use crate::{get_type, parse_path, LogMessage, Progress}; use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; use std::str::FromStr; use std::sync::Arc; +use rayon::ThreadPool; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::watch::Sender; use crate::resource_pack::files::unknowntexture::UnknownTexture; @@ -25,9 +26,12 @@ pub fn parse_resource_pack_files( entries: &mut Vec<(String, Vec)>, progress: Sender, pack: Arc, + thread_pool: &ThreadPool ) { - entries.par_iter_mut().for_each(move |(name, content)| { - parse_resource_pack_file(logger, &progress, &pack, name, content); + thread_pool.install(|| { + entries.par_iter_mut().for_each(move |(name, content)| { + parse_resource_pack_file(logger, &progress, &pack, name, content); + }); }); } @@ -42,14 +46,14 @@ fn parse_resource_pack_file( current: name.to_string(), }); #[cfg(feature = "profiling")] - let _ = crate::profiler::profiler::ScopeTimer::new( + let _ = crate::profiler::ScopeTimer::new( if name.ends_with(".json") || name.ends_with(".mcmeta") { "parse_resource_pack_files::json" - } else if name.ends_with(".png") && crate::get_type(&name) == Some("textures") { + } else if name.ends_with(".png") && get_type(&name) == Some("textures") { "parse_resource_pack_files::texture" } else if name.ends_with(".vsh") || name.ends_with(".fsh") || name.ends_with(".glsl") { "parse_resource_pack_files::shader" - } else if name.ends_with(".ogg") && crate::get_type(&name) == Some("sounds") { + } else if name.ends_with(".ogg") && get_type(&name) == Some("sounds") { "parse_resource_pack_files::sound" } else { "parse_resource_pack_files::unknown" diff --git a/packobf/src/lib.rs b/packobf/src/lib.rs index 61c1156..bf5a45b 100644 --- a/packobf/src/lib.rs +++ b/packobf/src/lib.rs @@ -13,7 +13,7 @@ pub mod utils; use crate::cache::Cache; use crate::optimized_zip_writer::OptimizedZipWriter; -use crate::options::Options; +use crate::options::{Options, ShaderCompression}; use crate::resource_pack::files::atlas::Atlas; use crate::resource_pack::files::blockstate::Blockstate; use crate::resource_pack::files::font::Font; @@ -25,19 +25,22 @@ use crate::resource_pack::files::shader::Shader; use crate::resource_pack::files::sound::Sound; use crate::resource_pack::files::sound_definitions::SoundDefinitions; use crate::resource_pack::files::texture::Texture; +use crate::resource_pack::files::unknowntexture::UnknownTexture; use crate::resource_pack::identifier::Identifier; use crate::resource_pack::mapping; use crate::resource_pack::mapping::{IdUsageCounter, Mapping}; use crate::resource_pack::pack::ResourcePack; -use crate::LogLevel::Info; +use crate::LogLevel::{Info, Warning}; +use dashmap::DashMap; use rayon::prelude::*; -use std::io::{Cursor, Error, Read}; +use rayon::{ThreadPool, ThreadPoolBuilder}; +use std::error::Error; +use std::io::{Cursor, Read}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::watch::Sender; use zip::ZipArchive; -use crate::resource_pack::files::unknowntexture::UnknownTexture; pub fn process_zip( input_bytes: Vec, @@ -45,10 +48,10 @@ pub fn process_zip( progress: Sender, logger: &UnboundedSender, cache_file: &Option, -) -> zip::result::ZipResult> { +) -> Result, Box> { let _ = progress.send(Progress::Idle); #[cfg(feature = "profiling")] - profiler::profiler::PROFILER.store(Arc::new(profiler::profiler::Profiler::new())); + profiler::PROFILER.store(Arc::new(profiler::Profiler::new())); let progress_clone = progress.clone(); let reader = Cursor::new(&input_bytes); @@ -80,11 +83,16 @@ pub fn process_zip( let id_usage_counter = IdUsageCounter::default(); mapping::set_id_usage_counter(id_usage_counter); + let pool = ThreadPoolBuilder::new() + .num_threads(options.num_threads.unwrap_or(0)) + .build()?; + file_parser::parse_resource_pack_files( logger, &mut entries, progress.clone(), Arc::clone(&pack), + &pool, ); usage_checker::check_usage(logger, &pack); @@ -95,16 +103,20 @@ pub fn process_zip( } mapping::set_mappings(mapping); - let mut items = collect_files(pack); + let mut items = collect_files(pack, &pool); let total = items.len(); let mut output = Cursor::new(Vec::new()); let writer = OptimizedZipWriter::new(&mut output); - let counter = AtomicUsize::new(0); if options.block_unzipping { // Add this file first to make tools crash before they can read the data - writer.add_file("assets\0", Vec::new().as_slice(), options, &None)?; + writer.add_file( + "assets\0", + Vec::new().as_slice(), + options, + &None, + )?; // `\0` (null) is universally disallowed inside filenames, but Minecraft doesn't care } @@ -113,28 +125,97 @@ pub fn process_zip( level: Info, message: format!("Loading cache from {}", cache), }); - let cache = Cache::load_from_file(cache)?; - let _ = logger.send(LogMessage { - level: Info, - message: format!("Cache loaded: {} items", cache.items.len()), - }); + let cache = match Cache::load_from_file(cache) { + Ok(cache) => { + let _ = logger.send(LogMessage { + level: Info, + message: format!("Cache loaded: {} items", cache.items.len()), + }); + cache + } + Err(e) => { + let _ = logger.send(LogMessage { + level: Warning, + message: format!("Invalid cache file, creating a new one. Error: {}", e), + }); + Cache { + items: DashMap::new(), + } + } + }; Some(cache) } else { None }; - items.par_iter_mut().for_each(|(name, item)| { - match add_item_to_archive( - options, &progress, logger, total, &writer, &counter, &cache, name, item, - ) { - Ok(_) => {} - Err(e) => { - let _ = logger.send(LogMessage { - level: LogLevel::Error, - message: format!("Failed to add item to archive: {}", e), - }); + pool.install(|| { + let total_to_optimize = AtomicUsize::new(0); + let to_optimize: Vec<_> = items + .par_iter_mut() + .filter(|(_, item)| match item { + ResourcePackItem::Texture(_) => { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } + ResourcePackItem::UnknownTexture(_) => { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } + ResourcePackItem::Shader(_) => { + if options.shader_compression != ShaderCompression::None { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } else { + false + } + } + ResourcePackItem::Sound(_) => { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } + _ => false, + }) + .collect(); + + let total_to_optimize = to_optimize.len(); + let counter = AtomicUsize::new(0); + to_optimize.into_par_iter().for_each(|(name, item)| { + let _ = progress.send(Progress::Optimizing { + current: name.to_string(), + index: counter.fetch_add(1, Ordering::Relaxed), + total: total_to_optimize, + }); + match item { + ResourcePackItem::Texture(x) => { + x.unknown_texture.optimize(options, logger, &cache); + } + ResourcePackItem::UnknownTexture(x) => { + x.optimize(options, logger, &cache); + } + ResourcePackItem::Shader(x) => { + x.optimize(options, logger); + } + ResourcePackItem::Sound(x) => { + x.optimize(logger, &cache); + } + _ => {} } - } + }); + + let counter = AtomicUsize::new(0); + items.par_iter_mut().for_each(|(name, item)| { + match add_item_to_archive( + options, &progress, logger, total, &writer, &counter, &cache, name, item, + ) { + Ok(_) => {} + Err(e) => { + let _ = logger.send(LogMessage { + level: LogLevel::Error, + message: format!("Failed to add item to archive: {}", e), + }); + } + } + }); }); writer.finish()?; @@ -146,7 +227,7 @@ pub fn process_zip( let _ = progress.send(Progress::Done); #[cfg(feature = "profiling")] - profiler::profiler::PROFILER.load().print(); + profiler::PROFILER.load().print(); Ok(output.into_inner()) } @@ -161,7 +242,7 @@ fn add_item_to_archive( cache: &Option, name: &mut String, item: &mut ResourcePackItem, -) -> Result<(), Error> { +) -> Result<(), Box> { let _ = progress.send(Progress::Building { current: name.to_string(), index: counter.fetch_add(1, Ordering::Relaxed), @@ -185,13 +266,20 @@ fn add_item_to_archive( } } match item { - ResourcePackItem::Texture(o) => { - o.optimize(options, logger, cache); - writer.add_file(name.as_str(), o.unknown_texture.bytes.as_slice(), options, cache) - } + ResourcePackItem::Texture(o) => writer.add_file( + name.as_str(), + o.unknown_texture.bytes.as_slice(), + options, + cache, + ), ResourcePackItem::Shader(o) => { o.optimize(options, logger); - writer.add_file(name.as_str(), o.content.as_bytes(), options, cache) + writer.add_file( + name.as_str(), + o.content.as_bytes(), + options, + cache, + ) } ResourcePackItem::Json(o) => writer.add_file( name.as_str(), @@ -199,125 +287,155 @@ fn add_item_to_archive( options, cache, ), - ResourcePackItem::Model(o) => { - writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) - } - ResourcePackItem::Unknown(o) => { - writer.add_file(name.as_str(), o.bytes.as_slice(), options, cache) - } - ResourcePackItem::BlockStateDefinition(o) => { - writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) - } - ResourcePackItem::FontDefinition(o) => { - writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) - } - ResourcePackItem::ItemDefinition(o) => { - writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) - } - ResourcePackItem::Sound(o) => { - o.optimize(logger, cache); - writer.add_file(name.as_str(), o.bytes.as_slice(), options, cache) - } - ResourcePackItem::SoundDefinitions(o) => { - writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) - } - ResourcePackItem::Atlas(o) => { - writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) - } - ResourcePackItem::UnknownTexture(o) => { - o.optimize(options, logger, cache); - writer.add_file(name.as_str(), o.bytes.as_slice(), options, cache) - } + ResourcePackItem::Model(o) => writer.add_file( + name.as_str(), + o.to_string().as_bytes(), + options, + cache, + ), + ResourcePackItem::Unknown(o) => writer.add_file( + name.as_str(), + o.bytes.as_slice(), + options, + cache, + ), + ResourcePackItem::BlockStateDefinition(o) => writer.add_file( + name.as_str(), + o.to_string().as_bytes(), + options, + cache, + ), + ResourcePackItem::FontDefinition(o) => writer.add_file( + name.as_str(), + o.to_string().as_bytes(), + options, + cache, + ), + ResourcePackItem::ItemDefinition(o) => writer.add_file( + name.as_str(), + o.to_string().as_bytes(), + options, + cache, + ), + ResourcePackItem::Sound(o) => writer.add_file( + name.as_str(), + o.bytes.as_slice(), + options, + cache, + ), + ResourcePackItem::SoundDefinitions(o) => writer.add_file( + name.as_str(), + o.to_string().as_bytes(), + options, + cache, + ), + ResourcePackItem::Atlas(o) => writer.add_file( + name.as_str(), + o.to_string().as_bytes(), + options, + cache, + ), + ResourcePackItem::UnknownTexture(o) => writer.add_file( + name.as_str(), + o.bytes.as_slice(), + options, + cache, + ), }?; Ok(()) } -fn collect_files(pack: Arc) -> Vec<(String, ResourcePackItem)> { +fn collect_files( + pack: Arc, + thread_pool: &ThreadPool, +) -> Vec<(String, ResourcePackItem)> { profile_scope!("collect_files"); - let texture_iter = pack.textures.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Texture(kv.value().clone()), - ) - }); - let unknown_texture_iter = pack.unknown_textures.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::UnknownTexture(kv.value().clone()), - ) - }); - let shader_iter = pack.shaders.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Shader(kv.value().clone()), - ) - }); - let model_iter = pack.models.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Model(kv.value().clone()), - ) - }); - let json_iter = pack - .json_files - .par_iter() - .map(|kv| (kv.key().clone(), ResourcePackItem::Json(kv.value().clone()))); - let unknown_iter = pack.unknown_files.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Unknown(kv.value().clone()), - ) - }); - let blockstate_iter = pack.blockstates.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::BlockStateDefinition(kv.value().clone()), - ) - }); - let font_iter = pack.fonts.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::FontDefinition(kv.value().clone()), - ) - }); - let item_iter = pack.items.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::ItemDefinition(kv.value().clone()), - ) - }); - let sound_iter = pack.sounds.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Sound(kv.value().clone()), - ) - }); - let sound_definitions_iter = pack.sound_definitions.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::SoundDefinitions(kv.value().clone()), - ) - }); - let atlas_iter = pack.atlases.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Atlas(kv.value().clone()), - ) - }); + thread_pool.install(|| { + let texture_iter = pack.textures.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Texture(kv.value().clone()), + ) + }); + let unknown_texture_iter = pack.unknown_textures.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::UnknownTexture(kv.value().clone()), + ) + }); + let shader_iter = pack.shaders.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Shader(kv.value().clone()), + ) + }); + let model_iter = pack.models.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Model(kv.value().clone()), + ) + }); + let json_iter = pack + .json_files + .par_iter() + .map(|kv| (kv.key().clone(), ResourcePackItem::Json(kv.value().clone()))); + let unknown_iter = pack.unknown_files.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Unknown(kv.value().clone()), + ) + }); + let blockstate_iter = pack.blockstates.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::BlockStateDefinition(kv.value().clone()), + ) + }); + let font_iter = pack.fonts.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::FontDefinition(kv.value().clone()), + ) + }); + let item_iter = pack.items.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::ItemDefinition(kv.value().clone()), + ) + }); + let sound_iter = pack.sounds.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Sound(kv.value().clone()), + ) + }); + let sound_definitions_iter = pack.sound_definitions.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::SoundDefinitions(kv.value().clone()), + ) + }); + let atlas_iter = pack.atlases.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Atlas(kv.value().clone()), + ) + }); - texture_iter - .chain(unknown_texture_iter) - .chain(shader_iter) - .chain(model_iter) - .chain(json_iter) - .chain(unknown_iter) - .chain(blockstate_iter) - .chain(font_iter) - .chain(item_iter) - .chain(sound_iter) - .chain(sound_definitions_iter) - .chain(atlas_iter) - .collect() + texture_iter + .chain(unknown_texture_iter) + .chain(shader_iter) + .chain(model_iter) + .chain(json_iter) + .chain(unknown_iter) + .chain(blockstate_iter) + .chain(font_iter) + .chain(item_iter) + .chain(sound_iter) + .chain(sound_definitions_iter) + .chain(atlas_iter) + .collect() + }) } #[derive(Clone, Debug)] @@ -330,6 +448,11 @@ pub enum Progress { Parsing { current: String, }, + Optimizing { + current: String, + index: usize, + total: usize, + }, Building { current: String, index: usize, @@ -375,7 +498,7 @@ fn parse_path(path: &str) -> (String, Identifier) { let overlay = if path.starts_with("assets/") { "".to_string() } else { - parts.next().unwrap().to_string() + parts.next().unwrap_or("").to_string() }; parts.next(); // skip assets diff --git a/packobf/src/optimized_zip_writer.rs b/packobf/src/optimized_zip_writer.rs index 2f304e5..20369f2 100644 --- a/packobf/src/optimized_zip_writer.rs +++ b/packobf/src/optimized_zip_writer.rs @@ -1,28 +1,25 @@ use crate::cache::{Cache, ItemType}; -pub(crate) use crate::options::ZOPFLI_OPTIONS; -use crate::options::{Compression, Options}; +use crate::options::{analyze_and_get_zopfli_config_best, analyze_and_get_zopfli_config_normal, Compression, Options, PreCheckResult, ULTRA_ZOPFLI_OPTIONS}; use crate::profile_scope; use byteorder::{LittleEndian, WriteBytesExt}; -use crc32fast::Hasher as Crc32Hasher; use dashmap::DashMap; use libdeflater::CompressionLvl; use sha2::{Digest, Sha256}; use std::io::{self, Error, Seek, Write}; use std::sync::{Arc, Mutex}; +use crate::options::PreCheckResult::{CompressWithZopfli, Skip}; #[derive(Clone, Debug)] pub struct CachedFileData { pub header_offset: u32, - pub crc32: u32, + pub compression_method: u16, pub compressed_size: u32, - pub uncompressed_size: u32, } struct CentralDirectoryEntry { filename: String, - crc32: u32, + compression_method: u16, compressed_size: u32, - uncompressed_size: u32, header_offset: u32, } @@ -65,16 +62,20 @@ impl OptimizedZipWriter { return self.record_entry(&mut inner, filename, cached_data); } - - // Calculate CRC32 (Required for Central Directory) - let mut crc32_hasher = Crc32Hasher::new(); - crc32_hasher.update(data); - let crc32 = crc32_hasher.finalize(); let uncompressed_size = data.len() as u32; // Compress the data using DEFLATE - let compressed_data: Vec = Self::compress(data, options, &hash, cache)?; - let compressed_size = compressed_data.len() as u32; + let compressed_data = Self::compress(data, options, &hash, cache)?; + let mut compressed_size = compressed_data.len() as u32; + let using_store = compressed_size == 0; // Skip compression if it's larger when compressed + if using_store { + compressed_size = uncompressed_size; + } + let compression_method = if using_store { + 0 // Store + } else { + 8 // Deflate + }; let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); @@ -90,9 +91,9 @@ impl OptimizedZipWriter { // Write the Minimized Local File Header (LFH) // We zero out the metadata, filename length, and omit the filename string to save space. inner.writer.write_u32::(0x04034b50)?; // LFH Signature - inner.writer.write_u16::(10)?; // Version needed to extract (1.0) + inner.writer.write_u16::(0)?; // Version needed to extract (Zeroed) inner.writer.write_u16::(0)?; // General purpose bit flag - inner.writer.write_u16::(8)?; // Compression method (8 = Deflate) + inner.writer.write_u16::(0)?; // Compression method (Zeroed) inner.writer.write_u32::(0)?; // Last mod file time and date (Zeroed) inner.writer.write_u32::(0)?; // CRC-32 (Zeroed) inner.writer.write_u32::(0)?; // Compressed size (Zeroed) @@ -101,22 +102,24 @@ impl OptimizedZipWriter { inner.writer.write_u16::(0)?; // Extra field length (Zeroed) // Write the actual compressed payload - inner.writer.write_all(&compressed_data)?; + if using_store { + inner.writer.write_all(data)?; // When using Store + } else { + inner.writer.write_all(&compressed_data)?; // When using Deflate + } let new_cache = CachedFileData { header_offset, - crc32, - compressed_size, - uncompressed_size, + compression_method, + compressed_size }; // Insert into the hashmap so future identical files point here self.content_cache.insert(hash, new_cache.clone()); inner.cd_entries.push(CentralDirectoryEntry { filename: filename.to_string(), - crc32: new_cache.crc32, + compression_method: new_cache.compression_method, compressed_size: new_cache.compressed_size, - uncompressed_size: new_cache.uncompressed_size, header_offset: new_cache.header_offset, }); @@ -140,14 +143,19 @@ impl OptimizedZipWriter { return Ok(bytes); } } + let input_size = data.len(); Ok(match options.compression { - Compression::Simplest => { + Compression::Fastest => { let mut compressor = libdeflater::Compressor::default(); let mut out = vec![0u8; compressor.deflate_compress_bound(data.len())]; let size = compressor .deflate_compress(data, &mut out) .map_err(|_| Error::other("Compression failed"))?; out.truncate(size); + let out_size = out.len(); + if out_size > input_size { + out = vec![]; + } if let Some(cache) = cache { cache.add_item_hash( hash, @@ -158,31 +166,86 @@ impl OptimizedZipWriter { } out } - Compression::Normal => { + Compression::Fast => { let mut compressor = libdeflater::Compressor::new(CompressionLvl::best()); let mut out = vec![0u8; compressor.deflate_compress_bound(data.len())]; let size = compressor .deflate_compress(data, &mut out) .map_err(|_| Error::other("Compression failed"))?; out.truncate(size); + let out_size = out.len(); + if out_size > input_size { + out = vec![]; + } if let Some(cache) = cache { cache.add_item_hash( hash, &*out, - crate::cache::Compression::Normal as u8, + crate::cache::Compression::Fast as u8, ItemType::Generic, ) } out } - Compression::Max => { + Compression::Normal => { + let pre_check_result = analyze_and_get_zopfli_config_normal(data); + Self::compress_with_pre_check(data, hash, cache, input_size, pre_check_result)? + } + Compression::Best => { + let pre_check_result = analyze_and_get_zopfli_config_best(data); + Self::compress_with_pre_check(data, hash, cache, input_size, pre_check_result)? + } + Compression::Ultra => { let mut encoder = zopfli::DeflateEncoder::new( - ZOPFLI_OPTIONS.to_owned(), + ULTRA_ZOPFLI_OPTIONS.to_owned(), zopfli::BlockType::Dynamic, Vec::new(), ); encoder.write_all(data)?; - let out = encoder.finish()?; + let mut out = encoder.finish()?; + let out_size = out.len(); + if out_size > input_size { + out = vec![]; + } + if let Some(cache) = cache { + cache.add_item_hash( + hash, + &*out, + crate::cache::Compression::Ultra as u8, + ItemType::Generic, + ) + } + out + } + }) + } + + fn compress_with_pre_check(data: &[u8], hash: &[u8; 32], cache: &Option, input_size: usize, pre_check_result: PreCheckResult) -> Result, Error> { + Ok(match pre_check_result { + CompressWithZopfli(options) => { + let mut encoder = zopfli::DeflateEncoder::new( + options, + zopfli::BlockType::Dynamic, + Vec::new(), + ); + encoder.write_all(data)?; + let mut out = encoder.finish()?; + let out_size = out.len(); + if out_size > input_size { + out = vec![]; + } + if let Some(cache) = cache { + cache.add_item_hash( + hash, + &*out, + crate::cache::Compression::Best as u8, + ItemType::Generic, + ) + } + out + } + Skip => { + let out = vec![]; if let Some(cache) = cache { cache.add_item_hash( hash, @@ -204,9 +267,8 @@ impl OptimizedZipWriter { ) -> io::Result<()> { inner.cd_entries.push(CentralDirectoryEntry { filename: filename.to_string(), - crc32: data.crc32, + compression_method: data.compression_method, compressed_size: data.compressed_size, - uncompressed_size: data.uncompressed_size, header_offset: data.header_offset, }); Ok(()) @@ -229,14 +291,14 @@ impl OptimizedZipWriter { let filename_bytes = entry.filename.as_bytes(); writer.write_u32::(0x02014b50)?; // CD Signature - writer.write_u16::(10)?; // Version made by - writer.write_u16::(10)?; // Version needed to extract + writer.write_u16::(0)?; // Version made by + writer.write_u16::(0)?; // Version needed to extract writer.write_u16::(0)?; // General purpose bit flag - writer.write_u16::(8)?; // Compression method (8 = Deflate) + writer.write_u16::(entry.compression_method)?; // Compression method writer.write_u32::(0)?; // Last mod file time and date - writer.write_u32::(entry.crc32)?; // Actual CRC-32 + writer.write_u32::(0)?; // Actual CRC-32 writer.write_u32::(entry.compressed_size)?; // Actual Compressed size - writer.write_u32::(entry.uncompressed_size)?; // Actual Uncompressed size + writer.write_u32::(0)?; // Actual Uncompressed size writer.write_u16::(filename_bytes.len() as u16)?; // File name length writer.write_u16::(0)?; // Extra field length writer.write_u16::(0)?; // File comment length diff --git a/packobf/src/options.rs b/packobf/src/options.rs index f5687cc..6d1cd2b 100644 --- a/packobf/src/options.rs +++ b/packobf/src/options.rs @@ -1,6 +1,7 @@ use once_cell::sync::Lazy; use std::num::NonZeroU64; use clap::{Parser, ValueEnum}; +use libdeflater::{CompressionLvl, Compressor}; #[derive(Parser, Clone, Debug)] #[group(id = "options")] @@ -15,23 +16,38 @@ pub struct Options { pub block_unzipping: bool, #[arg(long)] pub corrupt_png_files: bool, + #[arg(long)] + pub num_threads: Option, } #[derive(ValueEnum, Clone, Debug)] pub enum Preset { - Simplest, + Fastest, + Fast, Normal, - Max, + Best, } impl Options { - pub fn simplest() -> Self { + pub fn fastest() -> Self { + Self { + compression: Compression::Fastest, + shader_compression: ShaderCompression::None, + rename_files: false, + block_unzipping: false, + corrupt_png_files: false, + num_threads: None, + } + } + + pub fn fast() -> Self { Self { - compression: Compression::Simplest, + compression: Compression::Fast, shader_compression: ShaderCompression::None, rename_files: false, block_unzipping: false, corrupt_png_files: false, + num_threads: None, } } @@ -42,24 +58,27 @@ impl Options { rename_files: false, block_unzipping: false, corrupt_png_files: false, + num_threads: None, } } - pub fn max() -> Self { + pub fn best() -> Self { Self { - compression: Compression::Max, + compression: Compression::Best, shader_compression: ShaderCompression::MinifyAndObfuscate, rename_files: true, block_unzipping: true, corrupt_png_files: true, + num_threads: None, } } pub fn from_preset(preset: Preset) -> Self { match preset { - Preset::Simplest => Self::simplest(), + Preset::Fastest => Self::fastest(), + Preset::Fast => Self::fast(), Preset::Normal => Self::normal(), - Preset::Max => Self::max(), + Preset::Best => Self::best(), } } } @@ -67,9 +86,11 @@ impl Options { #[repr(u8)] #[derive(ValueEnum, Clone, Debug)] pub enum Compression { - Simplest = 0, - Normal = 1, - Max = 2, + Fastest = 0, + Fast = 1, + Normal = 2, + Best = 3, + Ultra = 4, } #[repr(u8)] @@ -84,6 +105,132 @@ pub enum ShaderCompression { #[allow(clippy::unwrap_used)] pub static ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { iteration_count: NonZeroU64::new(25).unwrap(), - iterations_without_improvement: NonZeroU64::new(7).unwrap(), - maximum_block_splits: 50, + iterations_without_improvement: NonZeroU64::new(3).unwrap(), + maximum_block_splits: 15, +}); + +#[allow(clippy::unwrap_used)] +pub static FASTEST_ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { + iteration_count: NonZeroU64::new(3).unwrap(), + iterations_without_improvement: NonZeroU64::new(1).unwrap(), + maximum_block_splits: 2, +}); + +#[allow(clippy::unwrap_used)] +pub static FAST_ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { + iteration_count: NonZeroU64::new(5).unwrap(), + iterations_without_improvement: NonZeroU64::new(2).unwrap(), + maximum_block_splits: 5, +}); + +#[allow(clippy::unwrap_used)] +pub static NORMAL_ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { + iteration_count: NonZeroU64::new(12).unwrap(), + iterations_without_improvement: NonZeroU64::new(2).unwrap(), + maximum_block_splits: 10, +}); + +#[allow(clippy::unwrap_used)] +pub static SLOW_ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { + iteration_count: NonZeroU64::new(20).unwrap(), + iterations_without_improvement: NonZeroU64::new(3).unwrap(), + maximum_block_splits: 15, +}); + +#[allow(clippy::unwrap_used)] +pub static SLOWEST_ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { + iteration_count: NonZeroU64::new(25).unwrap(), + iterations_without_improvement: NonZeroU64::new(3).unwrap(), + maximum_block_splits: 15, +}); + +#[allow(clippy::unwrap_used)] +pub static ULTRA_ZOPFLI_OPTIONS: Lazy = Lazy::new(|| zopfli::Options { + iteration_count: NonZeroU64::new(40).unwrap(), + iterations_without_improvement: NonZeroU64::new(40).unwrap(), + maximum_block_splits: 25, }); + +pub enum PreCheckResult { + /// Skip Zopfli entirely + Skip, + /// Use Zopfli with dynamically assigned options + CompressWithZopfli(zopfli::Options), +} + +/// Pre-checks data compressibility using libdeflater (Level 9) +/// and dynamically calculates the Zopfli config for 'normal' preset. +pub fn analyze_and_get_zopfli_config_normal(data: &[u8]) -> PreCheckResult { + let (original_size, savings_ratio) = match analyze(data) { + Ok(value) => value, + Err(value) => return value, + }; + + // Less than 1% savings + if savings_ratio < 0.01 { + return PreCheckResult::Skip; // Don't waste CPU time on Zopfli + } + + // 1% to 8% savings + if savings_ratio < 0.08 { + return PreCheckResult::CompressWithZopfli(FASTEST_ZOPFLI_OPTIONS.to_owned()); + } + + // > 8% savings + PreCheckResult::CompressWithZopfli(match original_size { + 0..=51_200 => SLOW_ZOPFLI_OPTIONS.to_owned(), + + 51_201..=512_000 => NORMAL_ZOPFLI_OPTIONS.to_owned(), + + _ => FAST_ZOPFLI_OPTIONS.to_owned(), + }) +} + +/// Pre-checks data compressibility using libdeflater (Level 9) +/// and dynamically calculates the Zopfli config for 'best' preset. +pub fn analyze_and_get_zopfli_config_best(data: &[u8]) -> PreCheckResult { + let (original_size, savings_ratio) = match analyze(data) { + Ok(value) => value, + Err(value) => return value, + }; + + // Less than 1% savings + if savings_ratio < 0.01 { + return PreCheckResult::Skip; // Don't waste CPU time on Zopfli + } + + // 1% to 8% savings + if savings_ratio < 0.08 { + return PreCheckResult::CompressWithZopfli(FAST_ZOPFLI_OPTIONS.to_owned()); + } + + // > 8% savings + PreCheckResult::CompressWithZopfli(match original_size { + 0..=51_200 => SLOWEST_ZOPFLI_OPTIONS.to_owned(), + + 51_201..=512_000 => SLOW_ZOPFLI_OPTIONS.to_owned(), + + _ => NORMAL_ZOPFLI_OPTIONS.to_owned(), + }) +} + +fn analyze(data: &[u8]) -> Result<(usize, f64), PreCheckResult> { + let original_size = data.len(); + if original_size == 0 { + return Err(PreCheckResult::Skip); + } + + #[allow(clippy::unwrap_used)] + let mut compressor = Compressor::new(CompressionLvl::new(9).unwrap()); + let max_buf_len = compressor.deflate_compress_bound(original_size); + let mut compressed_buf = vec![0u8; max_buf_len]; + + let fast_compressed_size = match compressor.deflate_compress(data, &mut compressed_buf) { + Ok(sz) => sz, + Err(_) => return Err(PreCheckResult::Skip), + }; + + let bytes_saved = original_size.saturating_sub(fast_compressed_size); + let savings_ratio = bytes_saved as f64 / original_size as f64; + Ok((original_size, savings_ratio)) +} diff --git a/packobf/src/profiler.rs b/packobf/src/profiler.rs index 4003549..33a0fa8 100644 --- a/packobf/src/profiler.rs +++ b/packobf/src/profiler.rs @@ -15,6 +15,8 @@ pub static PROFILER: LazyLock> = LazyLock::new(|| ArcSwap::fro #[cfg(feature = "profiling")] pub struct Stat { pub total_ns: AtomicU64, + pub min_ns: AtomicU64, + pub max_ns: AtomicU64, pub calls: AtomicU64, } @@ -36,10 +38,14 @@ impl Profiler { let entry = self.stats.entry(name).or_insert_with(|| Stat { total_ns: AtomicU64::new(0), + min_ns: AtomicU64::new(nanos), + max_ns: AtomicU64::new(nanos), calls: AtomicU64::new(0), }); entry.total_ns.fetch_add(nanos, Ordering::Relaxed); + entry.min_ns.fetch_min(nanos, Ordering::Relaxed); + entry.max_ns.fetch_max(nanos, Ordering::Relaxed); entry.calls.fetch_add(1, Ordering::Relaxed); } @@ -49,9 +55,11 @@ impl Profiler { .iter() .map(|e| { let total = e.total_ns.load(Ordering::Relaxed); + let min = e.min_ns.load(Ordering::Relaxed); + let max = e.max_ns.load(Ordering::Relaxed); let calls = e.calls.load(Ordering::Relaxed); - (*e.key(), total, calls, total as f64 / calls.max(1) as f64) + (*e.key(), total, calls, total as f64 / calls.max(1) as f64, min, max) }) .collect::>(); @@ -59,13 +67,15 @@ impl Profiler { println!("==== PROFILING ===="); - for (name, total, calls, avg) in entries { + for (name, total, calls, avg, min, max) in entries { println!( - "{:<40} total={:>10.2}ms calls={:>8.2} avg={:>10.2}ยตs", + "{:<40} total={:>10.2}ms calls={:>8.2} avg={:>10.5}ms min={:>10.5}ms max={:>10.5}ms", name, total as f64 / 1_000_000.0, calls, - avg / 1_000.0 + avg / 1_000_000.0, + min as f64 / 1_000_000.0, + max as f64 / 1_000_000.0, ); } } @@ -98,6 +108,6 @@ impl Drop for ScopeTimer { macro_rules! profile_scope { ($name:expr) => { #[cfg(feature = "profiling")] - let _profiler_scope = $crate::profiler::profiler::ScopeTimer::new($name); + let _profiler_scope = $crate::profiler::ScopeTimer::new($name); }; } diff --git a/packobf/src/renamer.rs b/packobf/src/renamer.rs index 65e3b8d..6c2f27d 100644 --- a/packobf/src/renamer.rs +++ b/packobf/src/renamer.rs @@ -239,7 +239,7 @@ fn rebuild_atlas(pack: &ResourcePack) { atlas.sources.retain(|source| !matches!(source, Source::Directory { .. } | Source::Single { .. })); match atlas.atlas_type { AtlasType::Blocks => { - if atlas.overlay == "" { + if atlas.overlay.is_empty() { block_atlas_exists = true; } atlas.sources.push(Source::Directory { @@ -248,7 +248,7 @@ fn rebuild_atlas(pack: &ResourcePack) { }); } AtlasType::Items => { - if atlas.overlay == "" { + if atlas.overlay.is_empty() { item_atlas_exists = true; } atlas.sources.push(Source::Directory { diff --git a/packobf/src/resource_pack/files/texture.rs b/packobf/src/resource_pack/files/texture.rs index 29d2d52..cb84e04 100644 --- a/packobf/src/resource_pack/files/texture.rs +++ b/packobf/src/resource_pack/files/texture.rs @@ -31,8 +31,8 @@ impl Texture { ); let unknown_texture = UnknownTexture::new(path, bytes); Self { - overlay: overlay.into(), - identifier: identifier.into(), + overlay, + identifier, unknown_texture, } } diff --git a/packobf/src/resource_pack/files/unknowntexture.rs b/packobf/src/resource_pack/files/unknowntexture.rs index 316c5a2..df87201 100644 --- a/packobf/src/resource_pack/files/unknowntexture.rs +++ b/packobf/src/resource_pack/files/unknowntexture.rs @@ -71,12 +71,22 @@ impl UnknownTexture { return bytes; } } + let oxipng_options = match options.compression { - Compression::Simplest => &DEFAULT_OPTIONS, - Compression::Normal => &NORMAL_OPTIONS, - Compression::Max => &MAX_OPTIONS, + Compression::Fastest => FASTEST_OPTIONS.clone(), + Compression::Fast => FAST_OPTIONS.clone(), + Compression::Normal => { + NORMAL_OPTIONS.clone() + } + Compression::Best => { + BEST_OPTIONS.clone() + } + Compression::Ultra => { + ULTRA_OPTIONS.clone() + } }; - match optimize_from_memory(bytes, oxipng_options) { + + match optimize_from_memory(bytes, &oxipng_options) { Ok(value) => { if let Some(cache) = cache { cache.add_item( @@ -102,7 +112,7 @@ impl UnknownTexture { level: Info, message: format!("Image '{}' was recovered successfully.", path), }); - match optimize_from_memory(value.as_slice(), oxipng_options) { + match optimize_from_memory(value.as_slice(), &oxipng_options) { Ok(value) => { if let Some(cache) = cache { cache.add_item( @@ -140,14 +150,10 @@ impl UnknownTexture { } } } - } -/** -Currently, only the `deflater` option changes, but some other options could be modified based on the compression wanted. -*/ -// -static DEFAULT_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { +// +static FASTEST_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { fix_errors: true, force: false, filters: indexset! { @@ -175,6 +181,39 @@ static DEFAULT_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { scale_16: false, strip: StripChunks::All, deflater: Deflater::Libdeflater { compression: 6 }, // 6: default compression level + fast_evaluation: true, + timeout: Some(Duration::from_secs(3)), + max_decompressed_size: None, +}); + +static FAST_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { + fix_errors: true, + force: false, + filters: indexset! { + FilterStrategy::NONE, + FilterStrategy::SUB, + FilterStrategy::UP, + FilterStrategy::AVERAGE, + FilterStrategy::PAETH, + FilterStrategy::MinSum, + FilterStrategy::Entropy, + FilterStrategy::Bigrams, + FilterStrategy::BigEnt, + FilterStrategy::Brute { + num_lines: 8, + level: 12, + }, + }, + interlace: Some(false), + optimize_alpha: true, + bit_depth_reduction: true, + color_type_reduction: true, + palette_reduction: true, + grayscale_reduction: true, + idat_recoding: true, + scale_16: false, + strip: StripChunks::All, + deflater: Deflater::Libdeflater { compression: 12 }, // 12: max compression level for libdeflater fast_evaluation: false, timeout: Some(Duration::from_secs(3)), max_decompressed_size: None, @@ -207,13 +246,46 @@ static NORMAL_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { idat_recoding: true, scale_16: false, strip: StripChunks::All, - deflater: Deflater::Libdeflater { compression: 12 }, // 12: max compression level for libdeflater + deflater: Deflater::Zopfli(options::NORMAL_ZOPFLI_OPTIONS.to_owned()), fast_evaluation: false, timeout: Some(Duration::from_secs(3)), max_decompressed_size: None, }); -static MAX_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { +static BEST_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { + fix_errors: true, + force: false, + filters: indexset! { + FilterStrategy::NONE, + FilterStrategy::SUB, + FilterStrategy::UP, + FilterStrategy::AVERAGE, + FilterStrategy::PAETH, + FilterStrategy::MinSum, + FilterStrategy::Entropy, + FilterStrategy::Bigrams, + FilterStrategy::BigEnt, + FilterStrategy::Brute { + num_lines: 8, + level: 12, + }, + }, + interlace: Some(false), + optimize_alpha: true, + bit_depth_reduction: true, + color_type_reduction: true, + palette_reduction: true, + grayscale_reduction: true, + idat_recoding: true, + scale_16: false, + strip: StripChunks::All, + deflater: Deflater::Zopfli(options::SLOW_ZOPFLI_OPTIONS.to_owned()), + fast_evaluation: false, + timeout: None, + max_decompressed_size: None, +}); + +static ULTRA_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { fix_errors: true, force: false, filters: indexset! { @@ -240,10 +312,9 @@ static MAX_OPTIONS: Lazy = Lazy::new(|| oxipng::Options { idat_recoding: true, scale_16: false, strip: StripChunks::All, - deflater: Deflater::Zopfli(options::ZOPFLI_OPTIONS.to_owned()), // zopfli: best compression + deflater: Deflater::Zopfli(options::ULTRA_ZOPFLI_OPTIONS.to_owned()), fast_evaluation: false, timeout: Some(Duration::from_secs(3)), max_decompressed_size: None, }); // - diff --git a/packobf/src/shader_minifier/minifier.rs b/packobf/src/shader_minifier/minifier.rs index 06cf928..6eed1f4 100644 --- a/packobf/src/shader_minifier/minifier.rs +++ b/packobf/src/shader_minifier/minifier.rs @@ -238,11 +238,12 @@ impl VisitorMut for Minifier { false }; if !has_forbidden_qualifier { - let name = list.head.name.clone().unwrap().0.to_string(); - - if !name.starts_with("gl_") { - if let Some(short) = self.local_mapping.get(&name) { - list.head.name = Some(IdentifierData::from(short.as_str()).into()); + if let Some(identifier) = list.head.name.clone() { + let name = identifier.0.to_string(); + if !name.starts_with("gl_") { + if let Some(short) = self.local_mapping.get(&name) { + list.head.name = Some(IdentifierData::from(short.as_str()).into()); + } } } diff --git a/packobf_cli/Cargo.toml b/packobf_cli/Cargo.toml index 86bcced..31290b4 100644 --- a/packobf_cli/Cargo.toml +++ b/packobf_cli/Cargo.toml @@ -3,6 +3,7 @@ name = "packobf_cli" version = "0.2.1" edition = "2024" license = "MIT" +default-run = "packobf_cli" [dependencies] packobf = { path = "../packobf" } diff --git a/packobf_cli/src/main.rs b/packobf_cli/src/main.rs index fc66dd9..04cd183 100644 --- a/packobf_cli/src/main.rs +++ b/packobf_cli/src/main.rs @@ -32,6 +32,7 @@ struct Args { static LOOKING_GLASS: Emoji<'_, '_> = Emoji("๐Ÿ” ", ""); static TRUCK: Emoji<'_, '_> = Emoji("๐Ÿšš ", ""); static CLIP: Emoji<'_, '_> = Emoji("๐Ÿ”— ", ""); +static OPTIMIZING: Emoji<'_, '_> = Emoji("๐Ÿš€ ", ""); static BUILDING: Emoji<'_, '_> = Emoji("โš’๏ธ ", ""); static SPARKLE: Emoji<'_, '_> = Emoji("โœจ ", ":-)"); static CHECK: Emoji<'_, '_> = Emoji("โœ… ", "OK "); @@ -59,7 +60,7 @@ async fn main() { DecimalBytes(bytes.len() as u64), DecimalBytes(input_size as u64), DecimalBytes(bytes.len().abs_diff(input_size) as u64), - 100.0 - ratio * 100.0, + (100.0 - ratio * 100.0).abs(), if bytes.len() < input_size { "smaller" } else { @@ -77,7 +78,7 @@ pub async fn run_progress_loop( let global_started = Instant::now(); let mut stage_started = Instant::now(); let mut current_pb: Option = None; - // 0: Idle, 1: Reading, 2: Parsing, 4: Building + // 1: Idle, 2: Reading, 3: Parsing, 4: Optimizing, 5: Building let mut current_stage: u8 = 0; let clear_current = |pb: &mut Option| { @@ -136,7 +137,7 @@ pub async fn run_progress_loop( if current_stage != 1 { println!( "{} {} Initializing...", - style("[1/4]").bold().dim(), + style("[1/5]").bold().dim(), LOOKING_GLASS ); current_stage = 1; @@ -150,7 +151,7 @@ pub async fn run_progress_loop( clear_current(&mut current_pb); println!( "{} {} Reading archive...", - style("[2/4]").bold().dim(), + style("[2/5]").bold().dim(), TRUCK ); current_stage = 2; @@ -160,7 +161,7 @@ pub async fn run_progress_loop( let pb = current_pb.get_or_insert_with(|| { let p = ProgressBar::new(total as u64); p.set_style(bar_style.clone()); - p.set_prefix("[2/4]"); + p.set_prefix("[2/5]"); p }); if pb.position() < current as u64 { @@ -176,12 +177,12 @@ pub async fn run_progress_loop( clear_current(&mut current_pb); println!( "{} {} Parsing resource files...", - style("[3/4]").bold().dim(), + style("[3/5]").bold().dim(), CLIP ); let pb = ProgressBar::new_spinner(); pb.set_style(spinner_style.clone()); - pb.set_prefix("[3/4]"); + pb.set_prefix("[3/5]"); current_pb = Some(pb); current_stage = 3; stage_started = Instant::now(); @@ -193,7 +194,7 @@ pub async fn run_progress_loop( } } - Progress::Building { + Progress::Optimizing { current, index, total, @@ -201,20 +202,50 @@ pub async fn run_progress_loop( if current_stage != 4 { print_finished_stage(&mut current_pb, "Parsing Complete", stage_started); + clear_current(&mut current_pb); + println!( + "{} {} Optimizing resource pack...", + style("[4/5]").bold().dim(), + OPTIMIZING + ); + current_stage = 4; + stage_started = Instant::now(); + } + + let pb = current_pb.get_or_insert_with(|| { + let p = ProgressBar::new(total as u64); + p.set_style(bar_style.clone()); + p.set_prefix("[4/5]"); + p + }); + if pb.position() < index as u64 { + pb.set_position(index as u64); + } + pb.set_message(format!("File: {}", current)); + } + + Progress::Building { + current, + index, + total, + } => { + if current_stage != 5 { + print_finished_stage(&mut current_pb, "Optimizing Complete", stage_started); + clear_current(&mut current_pb); println!( "{} {} Building resource pack...", - style("[4/4]").bold().dim(), + style("[5/5]").bold().dim(), BUILDING ); - current_stage = 4; + current_stage = 5; stage_started = Instant::now(); } let pb = current_pb.get_or_insert_with(|| { let p = ProgressBar::new(total as u64); p.set_style(bar_style.clone()); - p.set_prefix("[4/4]"); + p.set_prefix("[5/5]"); p }); if pb.position() < index as u64 { diff --git a/packobf_gui/src/cxxqt_object.rs b/packobf_gui/src/cxxqt_object.rs index 3d798d0..aec1dcd 100644 --- a/packobf_gui/src/cxxqt_object.rs +++ b/packobf_gui/src/cxxqt_object.rs @@ -191,9 +191,10 @@ impl qobject::AppController { let options = Options { compression: match self.compression() { - 0 => Compression::Simplest, - 1 => Compression::Normal, - _ => Compression::Max, + 0 => Compression::Fastest, + 1 => Compression::Fast, + 2 => Compression::Normal, + _ => Compression::Best, }, shader_compression: match self.shader_compression() { 0 => ShaderCompression::None, @@ -203,6 +204,7 @@ impl qobject::AppController { rename_files: *self.rename_files(), block_unzipping: *self.block_unzipping(), corrupt_png_files: *self.corrupt_png_files(), + num_threads: None, }; let qt_thread = self.qt_thread(); @@ -241,6 +243,7 @@ impl qobject::AppController { Progress::Idle => "Idle".to_string(), Progress::ReadingZip { current, total } => format!("Reading ZIP ({}/{})", current, total), Progress::Parsing { current } => format!("Parsing {}", current), + Progress::Optimizing { current, index, total } => format!("Optimizing ({}/{}) {}", index, total, current), Progress::Building { current, index, total } => format!("Building ({}/{}) {}", index, total, current), Progress::Done => "Done".to_string(), };