Codex/compat next audit

.github/workflows/validate.yml

@@ -49,6 +49,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Restore mcpp registry cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.mcpp/registry
+          key: mcpp-registry-${{ runner.os }}-0.0.41-${{ hashFiles('pkgs/**/*.lua', 'tests/smoke_compat_*.sh', '.github/workflows/validate.yml') }}
+          restore-keys: |
+            mcpp-registry-${{ runner.os }}-0.0.41-
       - name: Download mcpp
         env:
           MCPP_VERSION: "0.0.41"
@@ -57,11 +64,16 @@ jobs:
             "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_VERSION}/mcpp-${MCPP_VERSION}-linux-x86_64.tar.gz"
           tar -xzf mcpp.tar.gz
           root="$PWD/mcpp-${MCPP_VERSION}-linux-x86_64"
+          mkdir -p "$HOME/.mcpp/registry"
+          cp -a "$root/registry/." "$HOME/.mcpp/registry/"
           echo "MCPP=$root/bin/mcpp" >> "$GITHUB_ENV"
           echo "MCPP_VENDORED_XLINGS=$root/registry/bin/xlings" >> "$GITHUB_ENV"
           echo "$root/bin" >> "$GITHUB_PATH"
       - name: Run compat smoke tests
+        env:
+          MCPP_INDEX_SMOKE_MCPP_HOME: ${{ runner.temp }}/mcpp-smoke-home
         run: |
+          mkdir -p "$MCPP_INDEX_SMOKE_MCPP_HOME"
           "$MCPP" --version
           timeout 1800 bash tests/smoke_compat_core.sh
           timeout 1800 bash tests/smoke_compat_imgui.sh
@@ -90,6 +102,13 @@ jobs:
             xlings: registry/bin/xlings.exe
     steps:
       - uses: actions/checkout@v4
+      - name: Restore mcpp registry cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.mcpp/registry
+          key: mcpp-registry-${{ runner.os }}-0.0.41-${{ hashFiles('pkgs/**/*.lua', 'tests/smoke_compat_*.sh', '.github/workflows/validate.yml') }}
+          restore-keys: |
+            mcpp-registry-${{ runner.os }}-0.0.41-
       - name: Download mcpp
         shell: bash
         env:
@@ -109,6 +128,8 @@ jobs:
           esac
 
           root="$PWD/$MCPP_ROOT"
+          mkdir -p "$HOME/.mcpp/registry"
+          cp -a "$root/registry/." "$HOME/.mcpp/registry/"
           if [[ "$RUNNER_OS" == "Windows" ]]; then
             echo "MCPP=$(cygpath -m "$root/${{ matrix.mcpp }}")" >> "$GITHUB_ENV"
             echo "MCPP_VENDORED_XLINGS=$(cygpath -m "$root/${{ matrix.xlings }}")" >> "$GITHUB_ENV"

pkgs/c/compat.khrplatform.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.khrplatform",
     description = "Khronos KHR platform header for OpenGL/EGL compat packages",
-    licenses    = {"MIT"},
+    licenses    = {"Khronos"},
     repo        = "https://github.com/KhronosGroup/EGL-Registry",
     type        = "package",
 

pkgs/c/compat.libarchive.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.libarchive",
     description = "Multi-format archive and compression library",
-    licenses    = {"BSD-2-Clause"},
+    licenses    = {"BSD-2-Clause", "BSD-3-Clause", "BSD-2-Clause-NetBSD", "CC0-1.0", "OpenSSL", "Apache-2.0", "public-domain"},
     repo        = "https://libarchive.org/",
     type        = "package",
 

pkgs/c/compat.x11.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.x11",
     description = "Xlib runtime library and public headers built from upstream sources",
-    licenses    = {"MIT"},
+    licenses    = {"BSD-1-Clause", "HPND-sell-variant", "ISC", "MIT", "MIT-Open-Group", "X11"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libx11",
     type        = "package",
 

pkgs/c/compat.xau.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xau",
     description = "X authorization file management library",
-    licenses    = {"MIT"},
+    licenses    = {"MIT-Open-Group"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxau",
     type        = "package",
 

pkgs/c/compat.xcursor.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xcursor",
     description = "X cursor management runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"HPND-sell-variant"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxcursor",
     type        = "package",
 

pkgs/c/compat.xdmcp.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xdmcp",
     description = "X Display Manager Control Protocol library",
-    licenses    = {"MIT"},
+    licenses    = {"MIT-Open-Group"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxdmcp",
     type        = "package",
 

pkgs/c/compat.xext.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xext",
     description = "X11 miscellaneous extensions runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"HPND-sell-variant", "ISC", "MIT", "MIT-Open-Group"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxext",
     type        = "package",
 

pkgs/c/compat.xfixes.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xfixes",
     description = "X Fixes extension runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"HPND-sell-variant"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxfixes",
     type        = "package",
 

pkgs/c/compat.xi.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xi",
     description = "X Input extension runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"MIT-Open-Group"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxi",
     type        = "package",
 

pkgs/c/compat.xinerama.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xinerama",
     description = "Xinerama extension runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"MIT-Open-Group"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxinerama",
     type        = "package",
 

pkgs/c/compat.xorgproto.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xorgproto",
     description = "X.Org protocol headers for X11-based compat packages",
-    licenses    = {"MIT"},
+    licenses    = {},
     repo        = "https://gitlab.freedesktop.org/xorg/proto/xorgproto",
     type        = "package",
 

pkgs/c/compat.xrandr.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xrandr",
     description = "X Resize, Rotate and Reflect extension runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"HPND-sell-variant"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxrandr",
     type        = "package",
 

pkgs/c/compat.xrender.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xrender",
     description = "X Rendering extension runtime library and public headers",
-    licenses    = {"MIT"},
+    licenses    = {"HPND-sell-variant"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxrender",
     type        = "package",
 

pkgs/c/compat.xtrans.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xtrans",
     description = "X.Org transport layer support headers used by Xlib",
-    licenses    = {"MIT"},
+    licenses    = {"HPND-sell-variant", "MIT", "MIT-Open-Group"},
     repo        = "https://gitlab.freedesktop.org/xorg/lib/libxtrans",
     type        = "package",
 

pkgs/c/compat.xz.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.xz",
     description = "XZ Utils liblzma compression library",
-    licenses    = {"0BSD", "LGPL-2.1-or-later"},
+    licenses    = {"0BSD"},
     repo        = "https://github.com/tukaani-project/xz",
     type        = "package",
 
@@ -79,6 +79,11 @@ package = {
             "-DHAVE_DECODER_SPARC=1",
             "-DHAVE_DECODER_RISCV=1",
             "-DHAVE_LZIP_DECODER=1",
+            "-DHAVE_MF_HC3=1",
+            "-DHAVE_MF_HC4=1",
+            "-DHAVE_MF_BT2=1",
+            "-DHAVE_MF_BT3=1",
+            "-DHAVE_MF_BT4=1",
         },
         sources = {
             "*/src/common/tuklib_physmem.c",

pkgs/c/compat.zstd.lua

@@ -3,7 +3,7 @@ package = {
     namespace   = "compat",
     name        = "compat.zstd",
     description = "Zstandard real-time compression algorithm",
-    licenses    = {"BSD-3-Clause"},
+    licenses    = {"BSD-3-Clause", "GPL-2.0-only"},
     repo        = "https://github.com/facebook/zstd",
     type        = "package",
 

tests/smoke_compat_archive.sh

@@ -14,11 +14,19 @@ if [[ -z "$MCPP_BIN" || ! -x "$MCPP_BIN" ]]; then
 fi
 
 TMP="$(mktemp -d)"
-trap 'rm -rf "$TMP"' EXIT
+if [[ "${MCPP_INDEX_KEEP_SMOKE_TMP:-0}" == "1" ]]; then
+    echo "KEEP: $TMP"
+else
+    trap 'rm -rf "$TMP"' EXIT
+fi
 SMOKE_CACHE_DIR="${MCPP_INDEX_SMOKE_CACHE_DIR:-}"
 SMOKE_XPKGS_DIR="${MCPP_INDEX_SMOKE_XPKGS_DIR:-}"
 
-export MCPP_HOME="$TMP/mcpp-home"
+if [[ -n "${MCPP_INDEX_SMOKE_MCPP_HOME:-}" ]]; then
+    export MCPP_HOME="$MCPP_INDEX_SMOKE_MCPP_HOME"
+else
+    export MCPP_HOME="$TMP/mcpp-home"
+fi
 mkdir -p "$MCPP_HOME"
 
 USER_MCPP="${HOME}/.mcpp"
@@ -102,6 +110,153 @@ int main() {
 }
 EOF
 
+"$MCPP_BIN" build
+"$MCPP_BIN" run
+
+mkdir -p "$TMP/compat-compression-standalone-smoke/src"
+cd "$TMP/compat-compression-standalone-smoke"
+cat > mcpp.toml <<EOF
+[package]
+name = "compat-compression-standalone-smoke"
+version = "0.1.0"
+
+[toolchain]
+default = "gcc@16.1.0"
+
+[indices]
+compat = { path = "$ROOT" }
+
+[dependencies.compat]
+zlib = "1.3.2"
+bzip2 = "1.0.8"
+lz4 = "1.10.0"
+xz = "5.8.3"
+zstd = "1.5.7"
+
+[targets.compat-compression-standalone-smoke]
+kind = "bin"
+main = "src/main.cpp"
+EOF
+
+if [[ -n "$SMOKE_CACHE_DIR" && -d "$SMOKE_CACHE_DIR" ]]; then
+    mkdir -p .mcpp/.xlings/data/runtimedir
+    find "$SMOKE_CACHE_DIR" -maxdepth 1 -type f \
+        \( -name '*.tar.gz' -o -name '*.tar.xz' -o -name '*.zip' \) \
+        -exec cp -f {} .mcpp/.xlings/data/runtimedir/ \;
+fi
+
+cat > src/main.cpp <<'EOF'
+#include <bzlib.h>
+#include <lz4.h>
+#include <lzma.h>
+#include <zlib.h>
+#include <zstd.h>
+
+#include <cstdint>
+#include <cstring>
+#include <limits>
+#include <vector>
+
+static bool same_bytes(const void* actual, size_t actual_size,
+                       const void* expected, size_t expected_size) {
+    return actual_size == expected_size &&
+           std::memcmp(actual, expected, expected_size) == 0;
+}
+
+int main() {
+    const uint8_t input[] = "mcpp compat compression smoke";
+    const size_t input_size = sizeof(input) - 1;
+
+    uint8_t zlib_compressed[256] = {};
+    uLongf zlib_compressed_size = sizeof(zlib_compressed);
+    if (compress2(zlib_compressed, &zlib_compressed_size, input,
+                  static_cast<uLong>(input_size), Z_BEST_SPEED) != Z_OK) {
+        return 1;
+    }
+    uint8_t zlib_output[sizeof(input)] = {};
+    uLongf zlib_output_size = input_size;
+    if (uncompress(zlib_output, &zlib_output_size, zlib_compressed,
+                   zlib_compressed_size) != Z_OK ||
+        !same_bytes(zlib_output, zlib_output_size, input, input_size)) {
+        return 2;
+    }
+
+    char bzip2_compressed[256] = {};
+    unsigned int bzip2_compressed_size = sizeof(bzip2_compressed);
+    if (BZ2_bzBuffToBuffCompress(
+            bzip2_compressed, &bzip2_compressed_size,
+            const_cast<char*>(reinterpret_cast<const char*>(input)),
+            static_cast<unsigned int>(input_size), 1, 0, 30) != BZ_OK) {
+        return 3;
+    }
+    char bzip2_output[sizeof(input)] = {};
+    unsigned int bzip2_output_size = input_size;
+    if (BZ2_bzBuffToBuffDecompress(bzip2_output, &bzip2_output_size,
+                                   bzip2_compressed, bzip2_compressed_size,
+                                   0, 0) != BZ_OK ||
+        !same_bytes(bzip2_output, bzip2_output_size, input, input_size)) {
+        return 4;
+    }
+
+    char lz4_compressed[256] = {};
+    const int lz4_compressed_size =
+        LZ4_compress_default(reinterpret_cast<const char*>(input),
+                             lz4_compressed, static_cast<int>(input_size),
+                             sizeof(lz4_compressed));
+    if (lz4_compressed_size <= 0) {
+        return 5;
+    }
+    char lz4_output[sizeof(input)] = {};
+    const int lz4_output_size =
+        LZ4_decompress_safe(lz4_compressed, lz4_output, lz4_compressed_size,
+                            sizeof(lz4_output));
+    if (lz4_output_size < 0 ||
+        !same_bytes(lz4_output, static_cast<size_t>(lz4_output_size), input,
+                    input_size)) {
+        return 6;
+    }
+
+    std::vector<char> zstd_compressed(ZSTD_compressBound(input_size));
+    const size_t zstd_compressed_size =
+        ZSTD_compress(zstd_compressed.data(), zstd_compressed.size(), input,
+                      input_size, 1);
+    if (ZSTD_isError(zstd_compressed_size)) {
+        return 7;
+    }
+    std::vector<char> zstd_output(input_size);
+    const size_t zstd_output_size =
+        ZSTD_decompress(zstd_output.data(), zstd_output.size(),
+                        zstd_compressed.data(), zstd_compressed_size);
+    if (ZSTD_isError(zstd_output_size) ||
+        !same_bytes(zstd_output.data(), zstd_output_size, input, input_size)) {
+        return 8;
+    }
+
+    std::vector<uint8_t> xz_compressed(lzma_stream_buffer_bound(input_size));
+    size_t xz_compressed_pos = 0;
+    if (lzma_easy_buffer_encode(0, LZMA_CHECK_CRC64, nullptr, input,
+                                input_size, xz_compressed.data(),
+                                &xz_compressed_pos,
+                                xz_compressed.size()) != LZMA_OK) {
+        return 9;
+    }
+    uint64_t xz_memlimit = (std::numeric_limits<uint64_t>::max)();
+    size_t xz_input_pos = 0;
+    size_t xz_output_pos = 0;
+    std::vector<uint8_t> xz_output(input_size);
+    if (lzma_stream_buffer_decode(&xz_memlimit, 0, nullptr,
+                                  xz_compressed.data(), &xz_input_pos,
+                                  xz_compressed_pos, xz_output.data(),
+                                  &xz_output_pos, xz_output.size()) !=
+            LZMA_OK ||
+        !same_bytes(xz_output.data(), xz_output_pos, input, input_size)) {
+        return 10;
+    }
+
+    return 0;
+}
+EOF
+
 "$MCPP_BIN" build
 "$MCPP_BIN" run
 echo "OK"

tests/smoke_compat_core.sh

@@ -18,7 +18,11 @@ trap 'rm -rf "$TMP"' EXIT
 SMOKE_CACHE_DIR="${MCPP_INDEX_SMOKE_CACHE_DIR:-}"
 SMOKE_XPKGS_DIR="${MCPP_INDEX_SMOKE_XPKGS_DIR:-}"
 
-export MCPP_HOME="$TMP/mcpp-home"
+if [[ -n "${MCPP_INDEX_SMOKE_MCPP_HOME:-}" ]]; then
+    export MCPP_HOME="$MCPP_INDEX_SMOKE_MCPP_HOME"
+else
+    export MCPP_HOME="$TMP/mcpp-home"
+fi
 mkdir -p "$MCPP_HOME/registry/data/xpkgs"
 
 USER_MCPP="${HOME}/.mcpp"