mirror of
https://github.com/humbletim/firestorm-gha.git
synced 2026-08-14 08:53:07 +00:00
ReleaseFS_open devtime testing
This commit is contained in:
tim
@@ -29,6 +29,9 @@ jobs:
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install NSIS
|
||||
run: choco install nsis
|
||||
|
||||
- name: Install autobuild dependencies
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
@@ -38,13 +41,6 @@ jobs:
|
||||
echo "AUTOBUILD_VARIABLES_FILE=$PWD/build-variables/variables" >> $GITHUB_ENV
|
||||
echo "AUTOBUILD_VSVER=170" >> $GITHUB_ENV
|
||||
|
||||
- name: ~Setup tmate session
|
||||
if: always()
|
||||
# if: (failure() && inputs.pause != 'never') || inputs.pause == 'always'
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
with:
|
||||
limit-access-to-actor: true
|
||||
|
||||
- name: Configure (open source build)
|
||||
run: |
|
||||
autobuild configure -A 64 -c ReleaseFS_open -- --package -DLL_TESTS:BOOL=FALSE
|
||||
@@ -59,3 +55,10 @@ jobs:
|
||||
name: firestorm-installer
|
||||
path: build-*/newview/Release/*Setup.exe
|
||||
|
||||
- name: ~Setup tmate session
|
||||
if: always()
|
||||
# if: (failure() && inputs.pause != 'never') || inputs.pause == 'always'
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
with:
|
||||
limit-access-to-actor: true
|
||||
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Euo pipefail
|
||||
|
||||
source $ghash/gha.upload-artifact.bash
|
||||
|
||||
HERE=$(pwd -W 2>/dev/null || pwd)
|
||||
|
||||
config_name=${config_name:-Release}
|
||||
build_dir=${build_dir:-build-vc170-64}
|
||||
packages_dir=${packages_dir:-$build_dir/packages}
|
||||
source_dir=${source_dir:-indra}
|
||||
snapshot_dir=${snapshot_dir:-fs-test}
|
||||
|
||||
function git_kv_sha() {
|
||||
function _git_sha() {
|
||||
local path="$1"
|
||||
[[ "$path" =~ /[.]git ]] && path="$(dirname "$path")"
|
||||
test -e "$path" || { echo _err $? "could not determine .git root from $1" >&2 ; return 19; }
|
||||
git -C "$path" describe --always --first-parent --abbrev=7 || { echo _err $? "could not describe '$path'" >&2 ; return 20; }
|
||||
}
|
||||
for kv in $* ; do
|
||||
local k=${kv/=*/} v=${kv/*=/}
|
||||
echo $k=`_git_sha $v`
|
||||
done
|
||||
}
|
||||
|
||||
(
|
||||
echo upstream_rel=$(git -C $source_dir rev-list --count HEAD)
|
||||
git_kv_sha version_viewer_sha=$source_dir
|
||||
git_kv_sha version_fsvr_sha=$HERE
|
||||
echo base=$base
|
||||
echo config_name=$config_name
|
||||
echo build_dir=$build_dir
|
||||
echo packages_dir=$packages_dir
|
||||
echo source_dir=$source_dir
|
||||
echo snapshot_dir=$snapshot_dir
|
||||
) | tee $snapshot_dir/ReleaseFS_open.env >&2
|
||||
|
||||
source $snapshot_dir/ReleaseFS_open.env
|
||||
|
||||
mkdir -pv $snapshot_dir
|
||||
|
||||
snapshot_dir_abs=$(readlink -f $snapshot_dir)
|
||||
|
||||
cpsync() { cp -lunrp "$@" ; }
|
||||
|
||||
|
||||
###########################################################################
|
||||
###########################################################################
|
||||
echo "SNAPSHOT EMERGED OBJECT FILES..." >&2
|
||||
mkdir -pv "$snapshot_dir/objs"
|
||||
|
||||
# DEFINITION: Where is the source code relative to the build dir?
|
||||
# Based on standard SL viewer layout, it's usually parallel or up one level.
|
||||
# Adjust source_dir if your layout differs (e.g. $ghash/indra or ../indra)
|
||||
|
||||
if [ ! -d "$source_dir" ]; then
|
||||
echo "CRITICAL: Source root '$source_dir' not found. Cannot perform 1:1 mapping." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 1. BUILD THE SOURCE MAP (The Source of Truth)
|
||||
# We use an associative array to map "basename" -> "relative_path_from_root"
|
||||
# Entry format: [llapp]="newview/llapp.cpp"
|
||||
declare -A SOURCE_MAP
|
||||
|
||||
echo "Indexing source tree for 1:1 reconstruction..." >&2
|
||||
# Find source units. We include .cpp, .c, .glsl, etc if they compile to objs.
|
||||
# We exclude 'test' directories if you don't want unit test objs.
|
||||
while read -r rel_path; do
|
||||
# src_file = "../indra/newview/llapp.cpp"
|
||||
|
||||
# Clean path relative to source_dir for the 'devtime' structure
|
||||
# e.g. "newview/llapp.cpp"
|
||||
# rel_path=$(realpath --relative-to="$source_dir" "$src_file")
|
||||
|
||||
# Key = "llapp" (no extension)
|
||||
filename=$(basename "$rel_path")
|
||||
key="${filename%.*}"
|
||||
|
||||
SOURCE_MAP["$key"]="$rel_path"
|
||||
|
||||
done < <(cd "$source_dir" && find . -type f \( -name "*.cpp" -o -name "*.c" \) | grep -v "/test/")
|
||||
|
||||
# 2. HARVEST AND RESTORE OBJECTS
|
||||
# We ignore the folder they sit in (firestorm-bin.dir, etc). We only care about the file content.
|
||||
find "$build_dir" -name "*.obj" | while read -r obj_path; do
|
||||
|
||||
obj_name=$(basename "$obj_path") # llapp.obj
|
||||
base_name="${obj_name%.*}" # llapp
|
||||
|
||||
# 3. CONSULT THE MAP
|
||||
if [ -n "${SOURCE_MAP[$base_name]+x}" ]; then
|
||||
# We found a corresponding source file!
|
||||
src_rel_path="${SOURCE_MAP[$base_name]}" # e.g. "newview/llapp.cpp"
|
||||
src_dir=$(dirname "$src_rel_path") # e.g. "newview"
|
||||
|
||||
# 4. EXECUTE THE 1:1 RESTORATION
|
||||
# Dest: objs/newview/llapp.cpp.obj
|
||||
dest_dir="$snapshot_dir/objs/$src_dir"
|
||||
dest_file="$dest_dir/$(basename "$src_rel_path").obj"
|
||||
|
||||
mkdir -p "$dest_dir"
|
||||
|
||||
# Copy and rename to enforce strictly "SourceFileName.obj" convention
|
||||
cp -up "$obj_path" "$dest_file"
|
||||
else
|
||||
# Optional: Log orphans that don't match known source (generated files, etc.)
|
||||
dir_name=$(basename $(dirname "$obj_path"))
|
||||
target_name=${dir_name%.dir}
|
||||
dest_dir="$snapshot_dir/_orphan"
|
||||
mkdir -p "$dest_dir"
|
||||
# THE CRITICAL STEP: Rename basename.obj -> basename.cpp.obj
|
||||
base_obj_name=$(basename "$obj_path" .obj)
|
||||
dest_file="$dest_dir/${target_name}.${base_obj_name}.cpp.obj"
|
||||
# Copy with update (-u) and preserve attributes (-p)
|
||||
echo "[ORPHAN] '$obj_path' '$dest_file'" >&2
|
||||
cp -up "$obj_path" "$dest_file"
|
||||
fi
|
||||
done
|
||||
|
||||
build_dir_rel=$(realpath --relative-to="$HERE" $build_dir || echo $build_dir)
|
||||
cpsync `find $build_dir_rel -name llwebrtc.lib` $snapshot_dir/objs/ || { echo missing llwebrtc.lib >&2 ; exit 61; }
|
||||
cpsync `find $build_dir_rel -name llphysicsextensions*.lib` $snapshot_dir/objs/ || { echo missing llphysicsextensions*.lib >&2 ; exit 62; }
|
||||
cpsync `find $build_dir_rel -name media_plugin_base.lib` $snapshot_dir/objs/ || { echo missing media_plugin_base.lib >&2 ; exit 63; }
|
||||
|
||||
# find "$build_dir" -name "*.lib" | grep "/$config_name/" | while read -r lib_file; do
|
||||
# echo cp -up "$lib_file" "$snapshot_dir/objs/"
|
||||
# done
|
||||
|
||||
(
|
||||
cd $snapshot_dir
|
||||
find objs/ -name \*.obj -o -name \*.res | sed 's@^@${snapshot_dir}/@' > llobjs.rsp.in || exit 77
|
||||
cd ..
|
||||
)
|
||||
|
||||
###########################################################################
|
||||
echo "SNAPSHOT METADATA..." >&2
|
||||
mkdir -pv $snapshot_dir/metadata
|
||||
mkdir -pv $snapshot_dir/metadata/tmp
|
||||
|
||||
test -s $snapshot_dir/metadata/artifacts.tar.xz || tar -cJvf $snapshot_dir/metadata/artifacts.tar.xz `find $build_dir_rel -type f -name \*.tlog -o -name \*.vcxproj\* -o -name \*.h -o -name \*.txt | grep -v /packages`
|
||||
|
||||
# cp -ua env.d $snapshot_dir/metadata
|
||||
# cp -ua $nunja_dir $snapshot_dir/metadata/tmp
|
||||
# cp -ua $build_dir/msvc.nunja.env $snapshot_dir/metadata/tmp
|
||||
|
||||
# test ! -s fstuple.json || cp -av fstuple.json $snapshot_dir/metadata/
|
||||
cp -ua $build_dir/newview/packages-info.txt $build_dir/newview/build_info.json $snapshot_dir/metadata/
|
||||
# cp -ua $nunja_dir/viewer_version.txt $snapshot_dir/metadata/
|
||||
env | grep INPUT > $snapshot_dir/metadata/tmp/INPUT.env
|
||||
# env | grep -i version= > $snapshot_dir/metadata/tmp/version.env
|
||||
find $build_dir/ -type f > $snapshot_dir/metadata/tmp/build_dir.files
|
||||
|
||||
( cat /d/a/_temp/_runner_file_commands/step_summary_*-scrubbed > $snapshot_dir/metadata/summary.md ) || true
|
||||
# ( ninja -C $build_dir -t commands ${viewer_bin}-bin | grep -Eo '(")?[-]D[^ =]+(=[^ ]*)?\1?' | grep -vE '_EXPORTS$' | awk '!seen[$0]++' > $snapshot_dir/lldefines.rsp ) || true
|
||||
# cp -uav $nunja_dir/*defines.rsp $snapshot_dir/metadata/ 2>/dev/null || true
|
||||
|
||||
###########################################################################
|
||||
echo "SNAPSHOT PACKAGES..." >&2
|
||||
mkdir -pv $snapshot_dir/3p/lib
|
||||
for x in `ls -1 $packages_dir/lib/release | grep -v webrtc` ; do
|
||||
cpsync $packages_dir/lib/release/$x $snapshot_dir/3p/lib/
|
||||
done
|
||||
mkdir -pv $snapshot_dir/3p/include
|
||||
for x in `ls -1 $packages_dir/include| grep -v webrtc` ; do
|
||||
cpsync $packages_dir/include/$x $snapshot_dir/3p/include/
|
||||
done
|
||||
cpsync $build_dir/newview/licenses.txt $snapshot_dir/3p/
|
||||
cpsync $build_dir/newview/packages-info.txt $snapshot_dir/3p/
|
||||
|
||||
###########################################################################
|
||||
echo "SNAPSHOT CORRESPONDING SOURCE..." >&2
|
||||
mkdir -pv $snapshot_dir/source
|
||||
cp -ua $source_dir/../LICENSE $snapshot_dir/
|
||||
|
||||
if true; then #[[ $viewer_bin == firestorm ]] ; then
|
||||
cp -ua $build_dir/newview/fsversionvalues.h $snapshot_dir/source/ || true
|
||||
fi
|
||||
cp -ua $build_dir/newview/viewerRes.rc $snapshot_dir/source/ || true
|
||||
|
||||
if true; then #[[ $viewer_bin == secondlife ]] ; then
|
||||
cpsync $packages_dir/llphysicsextensions* $snapshot_dir/source || true
|
||||
fi
|
||||
|
||||
(
|
||||
cd $source_dir
|
||||
find ~+ -name \*.cpp -o -name \*.inl -o -name \*.h -o -name \*.hpp \
|
||||
| grep -vE '/tests?/' > $snapshot_dir_abs/metadata/tmp/primary.source.txt
|
||||
time (
|
||||
tar -cf - -T $snapshot_dir_abs/metadata/tmp/primary.source.txt --show-transformed-names \
|
||||
--transform "s|^${PWD#/}/||" \
|
||||
2>/dev/null \
|
||||
| tar -xf - -C$snapshot_dir_abs/source #xz -T0 - -c > $snapshot_dir/includes.tar.xz
|
||||
)
|
||||
cd ..
|
||||
)
|
||||
|
||||
for x in `grep -Eo '[^"]+[.](cur|ico)' $build_dir/newview/viewerRes.rc | sort -u ` ; do
|
||||
if [[ -f $source_dir/newview/res/$x ]]; then
|
||||
echo "$build_dir/newview/viewerRes.rc::$x found in $source_dir/newview/res/ ..." >&2
|
||||
cp -unrp $source_dir/newview/res/$x $snapshot_dir/source/newview/res/
|
||||
elif [[ -f $build_dir/newview/$x ]]; then
|
||||
echo "$build_dir/newview/viewerRes.rc::$x found in $build_dir/newview/ ..." >&2
|
||||
cp -unrp $build_dir/newview/$x $snapshot_dir/source/newview/res/
|
||||
else
|
||||
echo "$build_dir/newview/viewerRes.rc::$x source icon not found..." >&2
|
||||
fi
|
||||
done
|
||||
|
||||
mkdir -pv $snapshot_dir/metadata/tmp/icons/
|
||||
for x in `ls $source_dir/newview/*/*.ico $source_dir/newview/*/*/*.ico` ; do
|
||||
cp -vua $x $snapshot_dir/metadata/tmp/icons/$(basename $(dirname $(dirname $x))).$(basename $(dirname $x)).$(basename $x)
|
||||
done
|
||||
|
||||
(
|
||||
cd $snapshot_dir
|
||||
for x in `ls source/* -1d` ; do
|
||||
if [[ -d "$x" ]]; then
|
||||
echo "$x" | sed 's@^@-I${snapshot_dir}/@'
|
||||
else
|
||||
echo "skipping non-directory source/ entry: $x" >&2
|
||||
fi
|
||||
done
|
||||
cd ..
|
||||
) > $snapshot_dir/llincludes.rsp.in
|
||||
|
||||
###########################################################################
|
||||
# stage installer/runtime
|
||||
|
||||
cat "`find $build_dir -name \*_setup_tmp.nsi`" | sed -e "s@^File [^ ]\+[/\\]newview[/\\]Release[/\\]@File @g;s@^File @File ${base}/runtime/@g;" > $snapshot_dir/metadata/tmp/runtime.installer.nsi
|
||||
|
||||
mkdir -pv $snapshot_dir/metadata/nsi/
|
||||
sed 's@"[^"]\+\\newview\\installers\\windows\\@\${snapshot_dir}/metadata/nsi/@g' $snapshot_dir/metadata/tmp/runtime.installer.nsi \
|
||||
> $snapshot_dir/metadata/installer.nsi.in
|
||||
grep -Eo '[^"]+\\newview\\installers\\windows\\[^"]+' $snapshot_dir/metadata/tmp/runtime.installer.nsi | tr '\\' '/' | sort -u | sed -e "s@[^ ]\\+/indra/@${source_dir}/@g;" > $build_dir/nsis.txt
|
||||
for x in `cat $build_dir/nsis.txt` ; do
|
||||
cp -unrp "$x" $snapshot_dir/metadata/nsi/
|
||||
done
|
||||
|
||||
APPLICATION_EXE=`find $build_dir -name Firestorm*.exe` || { echo "!APPLICATION_EXE" >&2 ; exit 209 ; }
|
||||
test -f "$APPLICATION_EXE" || { echo "!APPLICATION_EXE='$APPLICATION_EXE'" >&2 ; exit 225 ; }
|
||||
|
||||
grep -E ^File "$snapshot_dir/metadata/tmp/runtime.installer.nsi" | tr '\\' '/' | sed -e 's@^File @@g' | sort -u | fgrep -v "${APPLICATION_EXE}" > $build_dir/runtime.txt
|
||||
|
||||
head -2 $build_dir/runtime.txt
|
||||
cp -av $build_dir/runtime.txt $snapshot_dir/metadata/tmp/
|
||||
sed "s@$base/runtime/@\${snapshot_dir}/runtime/@g" $build_dir/runtime.txt > $snapshot_dir/metadata/runtime.rsp.in
|
||||
sed "s@$base/runtime/@runtime/@g" $build_dir/runtime.txt > $snapshot_dir/metadata/runtime.rsp
|
||||
head -2 $snapshot_dir/metadata/runtime.rsp.in
|
||||
|
||||
###########################################################################
|
||||
bundle=${base}-${upstream_rel}-${version_viewer_sha}-${version_fsvr_sha}
|
||||
echo "[7z] GENERATING ${bundle}-(devtime|runtime|snapshot).zip..." >&2
|
||||
|
||||
#cd $build_dir
|
||||
|
||||
#test ! -d $base/runtime || rm -v $base/runtime
|
||||
# package ${base:-fs-beta-7.1.12-e}/ => "devtime" capture
|
||||
time ${_7z:-7z} -mx5 -bd -tzip a ${bundle}-devtime.zip $base
|
||||
|
||||
# stage fs-beta-7.1.12-e/runtime/
|
||||
if test -x C:\\windows\\system32\\cmd.exe ; then
|
||||
cmd //c mklink //j "`echo $base/runtime | tr '/' '\\\\'`" "`echo $build_dir/newview/Release | tr '/' '\\\\'`"
|
||||
elif test ! -d $base/runtime ; then
|
||||
pwd >&2
|
||||
echo "ln -s -T $HERE/$build_dir/newview/Release $base/runtime" >&2
|
||||
ln -s -T $HERE/$build_dir/newview/Release $base/runtime
|
||||
fi
|
||||
time ${_7z:-7z} -mx5 -bd -tzip a ${bundle}-runtime.zip @$build_dir/runtime.txt
|
||||
|
||||
# make a copy of devtime and append @precision manifested runtime/ folder (to emerge a combined snapshot)
|
||||
cp -av ${bundle}-devtime.zip ${bundle}-snapshot.zip
|
||||
time ${_7z:-7z} -mx5 -bd -tzip a ${bundle}-snapshot.zip @$build_dir/runtime.txt
|
||||
|
||||
test "${_7z:-7z}" == 7z || { echo "NOUPLOAD 7z=${_7z}" >&2 ; exit 141 ; }
|
||||
echo "UPLOADING ARTIFACTS...${GITHUB_ACTIONS}" >&2
|
||||
gha-have-runtime || { echo "gha runtime unavailable" && exit 0 ; }
|
||||
grep gha-patch-upload-artifact /d/a/_actions/actions/upload-artifact/v4/dist/upload/index.js || gha-patch-upload-artifact
|
||||
|
||||
zipUploadStream=${bundle}-devtime.zip gha-upload-artifact-fast ${bundle}-devtime ${bundle}-devtime.zip 7
|
||||
zipUploadStream=${bundle}-runtime.zip gha-upload-artifact-fast ${bundle}-runtime ${bundle}-runtime.zip 7
|
||||
zipUploadStream=${bundle}-snapshot.zip gha-upload-artifact-fast ${bundle}-snapshot ${bundle}-snapshot.zip 7
|
||||
|
||||
# UPLOAD SNAPSHOT
|
||||
|
||||
# cd $snapshot_dir
|
||||
# gha-upload-artifact-fast ${version_full}-snapshot .
|
||||
Executable
+291
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
import glob
|
||||
import shlex
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# --- CONFIGURATION ---
|
||||
BUILD_ROOT = "build-vc170-64"
|
||||
|
||||
# Path Abstractions: (Anchor -> Variable Replacement)
|
||||
# We scan for these anchors in order. First match wins.
|
||||
PATH_VARS = [
|
||||
("/indra/", "$source/"),
|
||||
("/packages/", "$packages/"),
|
||||
(f"/{BUILD_ROOT.lower()}/", "$build/"),
|
||||
]
|
||||
|
||||
# Flags that are known to accept an argument separated by a space
|
||||
# e.g. /D _UNICODE or /I "path/to/include"
|
||||
# We act "greedy" with these: if we see one, we peek at the next token.
|
||||
GREEDY_FLAGS = {
|
||||
'/D', '/I', '/EXTERNAL:I',
|
||||
'/Fo', '/FO', '/Fd', '/FD', '/Fe', '/FE', '/FI', '/Fi', '/Fp', '/FP',
|
||||
'/YC', '/YU',
|
||||
}
|
||||
|
||||
class PathSanitizer:
|
||||
@staticmethod
|
||||
def clean_path(path_str):
|
||||
if not path_str: return ""
|
||||
# 1. Normalize Slashes & Case
|
||||
s = path_str.replace("\\", "/").lower()
|
||||
# 2. Variable Substitution
|
||||
for anchor, variable in PATH_VARS:
|
||||
idx = s.find(anchor)
|
||||
if idx != -1:
|
||||
suffix = s[idx+len(anchor):]
|
||||
return f"{variable}{suffix}"
|
||||
return s
|
||||
|
||||
@staticmethod
|
||||
def format_flag(flag, value=None):
|
||||
"""
|
||||
Recombines flag and value into a standardized string.
|
||||
Option: '/D _UNICODE' (Standardized spacing)
|
||||
"""
|
||||
# If the flag itself contained the value (e.g. /D_UNICODE), 'value' is None
|
||||
# We need to split them if we want to normalize the path inside.
|
||||
|
||||
# But our tokenizer splits them for us.
|
||||
# So 'flag' is the switch (/I) and 'value' is the payload (path).
|
||||
|
||||
if value is None:
|
||||
return flag
|
||||
|
||||
# Clean the value (it might be a path or a macro)
|
||||
# We only strictly normalize paths, but lowercasing macros is risky?
|
||||
# User accepted '/D _UNICODE', so we won't lowercase the macro value, only paths.
|
||||
|
||||
is_path_flag = flag.upper().startswith(("/I", "/EXTERNAL:I", "/FO", "/FD", "/FI", "/YC", "/YU"))
|
||||
|
||||
if is_path_flag:
|
||||
cleaned_val = PathSanitizer.clean_path(value)
|
||||
else:
|
||||
# It's a Define or Option, keep strict fidelity for the value
|
||||
cleaned_val = value
|
||||
|
||||
# Smart Quote: Only quote if space exists in the cleaned value
|
||||
if " " in cleaned_val:
|
||||
return f'{flag} "{cleaned_val}"'
|
||||
else:
|
||||
return f'{flag} {cleaned_val}'
|
||||
|
||||
class CompileUnit:
|
||||
def __init__(self, raw_source, raw_args):
|
||||
# Clean source path for display
|
||||
full_clean = PathSanitizer.clean_path(raw_source.strip())
|
||||
self.name = full_clean.replace("$source/", "").replace("$build/", "")
|
||||
self.flags = self._process_flags(raw_args)
|
||||
|
||||
def _process_flags(self, arg_string):
|
||||
# 1. Basic Tokenization using shlex to respect quotes
|
||||
# windows paths with backslashes can confuse shlex, so we escape them first?
|
||||
# Actually, tlog usually quotes paths with spaces.
|
||||
# Let's try a simple split first, but respecting quotes is hard without shlex.
|
||||
# Fallback: simple split by space, then repair quotes?
|
||||
# Given the "no spaces in filenames" constraint, simple split is safe IF quotes are balanced.
|
||||
|
||||
# However, "D:\foo bar" exists in Windows.
|
||||
# Let's use a custom generator that handles quoted strings.
|
||||
|
||||
tokens = []
|
||||
current_token = []
|
||||
in_quote = False
|
||||
|
||||
for char in arg_string:
|
||||
if char == '"':
|
||||
in_quote = not in_quote
|
||||
current_token.append(char)
|
||||
elif char == ' ' and not in_quote:
|
||||
if current_token:
|
||||
tokens.append("".join(current_token))
|
||||
current_token = []
|
||||
else:
|
||||
current_token.append(char)
|
||||
if current_token:
|
||||
tokens.append("".join(current_token))
|
||||
|
||||
# 2. Stateful Parsing (combining greedy flags)
|
||||
final_flags = []
|
||||
i = 0
|
||||
while i < len(tokens):
|
||||
token = tokens[i]
|
||||
|
||||
# Check if this token is a flag
|
||||
if token.startswith("/") or token.startswith("-"):
|
||||
# Clean quotes from the flag itself if present
|
||||
clean_token = token.replace('"', '')
|
||||
upper_token = clean_token.upper()
|
||||
|
||||
# Case 1: /Flag"Value" or /FlagValue (Value attached)
|
||||
# We need to detect if the value is already inside
|
||||
# Heuristic: If length > 2 and it's /D... or /I...
|
||||
|
||||
# But wait, we want to handle `/D _UNICODE`
|
||||
# If clean_token is EXACTLY in GREEDY_FLAGS, look ahead.
|
||||
|
||||
if upper_token in GREEDY_FLAGS:
|
||||
# It is a bare flag like /D. Check next token.
|
||||
if i + 1 < len(tokens) and not tokens[i+1].startswith(("/", "-")):
|
||||
# Next token is the argument
|
||||
arg = tokens[i+1].replace('"', '') # Strip quotes from arg
|
||||
final_flags.append(PathSanitizer.format_flag(clean_token, arg))
|
||||
i += 2 # Skip both
|
||||
continue
|
||||
else:
|
||||
# No argument follows, or next is a flag.
|
||||
# Treat as bare flag? Or error? usually /D needs arg.
|
||||
# Assuming attached like /D_UNICODE but token split failed?
|
||||
final_flags.append(token)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Case 2: Attached value (/I"Path")
|
||||
# We need to split it to sanitize the path part
|
||||
found_attached = False
|
||||
for greedy in GREEDY_FLAGS:
|
||||
if upper_token.startswith(greedy) and len(upper_token) > len(greedy):
|
||||
# It has attached data
|
||||
val = clean_token[len(greedy):]
|
||||
final_flags.append(PathSanitizer.format_flag(clean_token[:len(greedy)], val))
|
||||
found_attached = True
|
||||
break
|
||||
|
||||
if not found_attached:
|
||||
# Just a normal flag (e.g. /W3, /nologo)
|
||||
final_flags.append(token)
|
||||
|
||||
i += 1
|
||||
else:
|
||||
# Token doesn't start with / or - ... weird.
|
||||
# Probably a source file or loose artifact. Ignore or log?
|
||||
i += 1
|
||||
|
||||
return sorted(final_flags)
|
||||
|
||||
def load_tlogs():
|
||||
units = []
|
||||
pattern = os.path.join(BUILD_ROOT, "**", "CL.command.*.tlog")
|
||||
files = glob.glob(pattern, recursive=True)
|
||||
|
||||
print(f"[*] Found {len(files)} TLOG files in {BUILD_ROOT}...")
|
||||
|
||||
for fpath in files:
|
||||
try:
|
||||
with open(fpath, 'r', encoding='utf-16') as f: content = f.read()
|
||||
except UnicodeError:
|
||||
with open(fpath, 'r', encoding='utf-8', errors='ignore') as f: content = f.read()
|
||||
|
||||
lines = content.splitlines()
|
||||
current_source = None
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
if line.startswith('^'):
|
||||
current_source = line[1:]
|
||||
elif current_source:
|
||||
units.append(CompileUnit(current_source, line))
|
||||
current_source = None
|
||||
|
||||
print(f"[*] Parsed {len(units)} total compilation units.")
|
||||
return units
|
||||
|
||||
def mode_audit(units):
|
||||
if not units: return
|
||||
|
||||
total = len(units)
|
||||
flag_counts = Counter()
|
||||
for u in units: flag_counts.update(u.flags)
|
||||
|
||||
threshold = total * 0.95
|
||||
standard_flags = {f for f, c in flag_counts.items() if c > threshold}
|
||||
|
||||
print(f"\n=== STANDARD CONFIGURATION ({len(standard_flags)} flags) ===")
|
||||
|
||||
def sort_key(x):
|
||||
u = x.upper()
|
||||
if u.startswith("/D"): return (0, x)
|
||||
if u.startswith("/I") or "INCLUDE" in u: return (1, x)
|
||||
return (2, x)
|
||||
|
||||
for f in sorted(standard_flags, key=sort_key):
|
||||
print(f" {f}")
|
||||
|
||||
print("\n=== DEVIATION REPORT ===")
|
||||
|
||||
deviation_groups = defaultdict(list)
|
||||
for u in units:
|
||||
added = [f for f in u.flags if f not in standard_flags]
|
||||
dropped = [f for f in standard_flags if f not in u.flags]
|
||||
|
||||
# Filter noise (Output paths)
|
||||
meaningful_added = []
|
||||
for f in added:
|
||||
u_f = f.upper()
|
||||
if not (u_f.startswith("/FO") or u_f.startswith("/FD")):
|
||||
meaningful_added.append(f)
|
||||
|
||||
if meaningful_added or dropped:
|
||||
sig = (tuple(sorted(meaningful_added)), tuple(sorted(dropped)))
|
||||
deviation_groups[sig].append(u.name)
|
||||
|
||||
print(f"Found {len(deviation_groups)} unique deviation signatures.")
|
||||
|
||||
sorted_groups = sorted(deviation_groups.items(), key=lambda x: len(x[1]), reverse=True)
|
||||
|
||||
for (added, dropped), filenames in sorted_groups:
|
||||
example_count = 3
|
||||
examples = filenames[:example_count]
|
||||
remaining = len(filenames) - example_count
|
||||
|
||||
print(f"\n--- Group: {len(filenames)} files")
|
||||
print(f" (e.g. {', '.join(examples)}" + (f", ...)" if remaining > 0 else ")"))
|
||||
|
||||
if added:
|
||||
print(f" + ADDED:")
|
||||
for f in added: print(f" {f}")
|
||||
if dropped:
|
||||
print(f" - MISSING:")
|
||||
for f in dropped: print(f" {f}")
|
||||
|
||||
def mode_inspect(units, pattern):
|
||||
matches = [u for u in units if pattern.lower() in u.name.lower()]
|
||||
if not matches:
|
||||
print(f"No matches for '{pattern}'")
|
||||
return
|
||||
|
||||
target = matches[0]
|
||||
print(f"\n=== INSPECTION: {target.name} ===")
|
||||
|
||||
defines = []
|
||||
includes = []
|
||||
options = []
|
||||
|
||||
for f in target.flags:
|
||||
u = f.upper()
|
||||
if u.startswith("/D"): defines.append(f)
|
||||
elif u.startswith(("/I", "/EXTERNAL:I")): includes.append(f)
|
||||
else: options.append(f)
|
||||
|
||||
print("\n[DEFINES]")
|
||||
for f in sorted(defines): print(f" {f}")
|
||||
|
||||
print("\n[INCLUDES]")
|
||||
for f in sorted(includes): print(f" {f}")
|
||||
|
||||
print("\n[OPTIONS]")
|
||||
for f in sorted(options): print(f" {f}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 audit_compiler_v7.py [audit|inspect <filename>]")
|
||||
sys.exit(1)
|
||||
|
||||
all_units = load_tlogs()
|
||||
command = sys.argv[1]
|
||||
if command == "audit":
|
||||
mode_audit(all_units)
|
||||
elif command == "inspect":
|
||||
mode_inspect(all_units, sys.argv[2] if len(sys.argv) > 2 else "")
|
||||
Reference in New Issue
Block a user