mirror of
https://github.com/runkids/skillshare.git
synced 2026-11-19 02:17:09 +00:00
Extension and size filters could hide native executables and Python bytecode inside document, image and cache paths. Classify bounded headers before those filters, retain source boundaries, and report opaque and oversized file counts separately from auditable text. Keep archive signatures and uncertain headers informational rather than claiming runtime analysis. Honor analyzer selection for file and dashboard scans, and cover renamed payloads and benign twins with synthetic fixtures and fuzzing. Implements item 7 of #746. Co-authored-by: GPT-6.1 Sol <noreply@openai.com> Co-authored-by: sim <sim@example.invalid>
392 lines
13 KiB
Go
392 lines
13 KiB
Go
package audit
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/binary"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"unicode"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
const headerReadLimit = 512
|
||
|
||
// Final-release CPython 3.6–3.14 magic values, pinned to CPython v3.14.0:
|
||
// https://github.com/python/cpython/blob/v3.14.0/Include/internal/pycore_magic_number.h
|
||
// Review new values against a tagged CPython source and add a fixture before
|
||
// extending this table. This identifies a format, never decodes bytecode.
|
||
var pythonMagicVersions = map[uint16]string{
|
||
3379: "3.6", 3394: "3.7", 3413: "3.8", 3425: "3.9", 3439: "3.10",
|
||
3495: "3.11", 3531: "3.12", 3571: "3.13", 3627: "3.14",
|
||
}
|
||
|
||
var disguiseExtensions = map[string]bool{
|
||
".md": true, ".txt": true, ".rst": true, ".pdf": true,
|
||
".doc": true, ".docx": true, ".rtf": true, ".html": true,
|
||
".json": true, ".yaml": true, ".yml": true, ".toml": true,
|
||
".png": true, ".jpg": true, ".jpeg": true, ".gif": true,
|
||
".svg": true, ".webp": true, ".ico": true, ".bmp": true,
|
||
".tif": true, ".tiff": true,
|
||
}
|
||
|
||
// Cache payloads and content/extension mismatches are stronger evidence than
|
||
// dot names, which are common in tooling and virtual environments.
|
||
var disguiseCacheDirectories = map[string]bool{
|
||
".cache": true, "__pycache__": true,
|
||
}
|
||
|
||
func hiddenParent(path string) bool {
|
||
for _, part := range strings.Split(filepath.ToSlash(filepath.Dir(path)), "/") {
|
||
if strings.HasPrefix(part, ".") && part != "." {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func disguisedPath(path string) bool {
|
||
parts := strings.Split(filepath.ToSlash(path), "/")
|
||
for _, part := range parts[:len(parts)-1] {
|
||
if disguiseCacheDirectories[part] {
|
||
return true
|
||
}
|
||
}
|
||
return disguiseExtensions[strings.ToLower(filepath.Ext(path))]
|
||
}
|
||
|
||
func validELFHeader(data []byte) bool {
|
||
if len(data) < 24 || data[4] < 1 || data[4] > 2 || data[5] < 1 || data[5] > 2 || data[6] != 1 {
|
||
return false
|
||
}
|
||
class := data[4]
|
||
headerSize, sizeOffset := 52, 40
|
||
if class == 2 {
|
||
headerSize, sizeOffset = 64, 52
|
||
}
|
||
if len(data) < headerSize {
|
||
return false
|
||
}
|
||
var order binary.ByteOrder = binary.LittleEndian
|
||
if data[5] == 2 {
|
||
order = binary.BigEndian
|
||
}
|
||
return order.Uint16(data[16:18]) != 0 &&
|
||
order.Uint16(data[18:20]) != 0 &&
|
||
order.Uint32(data[20:24]) == 1 &&
|
||
int(order.Uint16(data[sizeOffset:sizeOffset+2])) == headerSize
|
||
}
|
||
|
||
func validZIPHeader(data []byte) bool {
|
||
switch {
|
||
case bytes.HasPrefix(data, []byte("PK\x03\x04")):
|
||
if len(data) < 30 {
|
||
return false
|
||
}
|
||
version := binary.LittleEndian.Uint16(data[4:6])
|
||
nameSize := uint64(binary.LittleEndian.Uint16(data[26:28]))
|
||
extraSize := uint64(binary.LittleEndian.Uint16(data[28:30]))
|
||
return version >= 10 && version <= 63 && nameSize > 0 && 30+nameSize+extraSize <= uint64(len(data))
|
||
case bytes.HasPrefix(data, []byte("PK\x05\x06")):
|
||
if len(data) < 22 {
|
||
return false
|
||
}
|
||
commentSize := uint64(binary.LittleEndian.Uint16(data[20:22]))
|
||
return binary.LittleEndian.Uint16(data[4:6]) == 0 &&
|
||
binary.LittleEndian.Uint16(data[6:8]) == 0 &&
|
||
binary.LittleEndian.Uint16(data[8:10]) == 0 &&
|
||
binary.LittleEndian.Uint16(data[10:12]) == 0 &&
|
||
binary.LittleEndian.Uint32(data[12:16]) == 0 &&
|
||
binary.LittleEndian.Uint32(data[16:20]) == 0 &&
|
||
22+commentSize <= uint64(len(data))
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// readHeader never follows links selected by the walker: callers must first
|
||
// require a regular file. No executable bit is used as evidence.
|
||
func readHeader(path string) ([]byte, error) {
|
||
f, err := os.Open(path)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer f.Close()
|
||
data, err := io.ReadAll(io.LimitReader(f, headerReadLimit))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// A PE header may be beyond the initial window. Read only the fixed
|
||
// COFF header and enough optional header bytes to validate the format.
|
||
if len(data) >= 64 && string(data[:2]) == "MZ" {
|
||
offset := int64(binary.LittleEndian.Uint32(data[60:64]))
|
||
if offset >= 64 && peOutsideWindow(data, uint64(offset)) {
|
||
extra := make([]byte, 24+112)
|
||
n, readErr := f.ReadAt(extra, offset)
|
||
if info, statErr := f.Stat(); statErr == nil && offset <= info.Size() && (readErr == nil || readErr == io.EOF) {
|
||
data = normalizePEWindow(data, extra[:n], uint64(info.Size())-uint64(offset))
|
||
}
|
||
}
|
||
}
|
||
return data, nil
|
||
}
|
||
|
||
// normalizePEWindow relocates the bounded PE validation bytes into the first
|
||
// window; the content itself is never executed or otherwise interpreted.
|
||
func normalizePEWindow(data, pe []byte, available uint64) []byte {
|
||
if len(data) < 64 || len(pe) < 26 || string(pe[:4]) != "PE\x00\x00" {
|
||
return data
|
||
}
|
||
optionalSize := binary.LittleEndian.Uint16(pe[20:22])
|
||
var min uint16
|
||
switch binary.LittleEndian.Uint16(pe[24:26]) {
|
||
case 0x10b:
|
||
min = 96
|
||
case 0x20b:
|
||
min = 112
|
||
case 0x107:
|
||
min = 56
|
||
}
|
||
if min == 0 || optionalSize < min || len(pe) < 24+int(min) || uint64(24)+uint64(optionalSize) > available {
|
||
return data
|
||
}
|
||
// The classifier checks the declared size; cap it to the validated prefix.
|
||
out := make([]byte, 64+24+int(min))
|
||
copy(out, data[:64])
|
||
copy(out[64:], pe[:24+int(min)])
|
||
binary.LittleEndian.PutUint32(out[60:64], 64)
|
||
binary.LittleEndian.PutUint16(out[84:86], min)
|
||
return out
|
||
}
|
||
|
||
func peOutsideWindow(data []byte, offset uint64) bool {
|
||
if offset+26 > headerReadLimit {
|
||
return true
|
||
}
|
||
if offset+22 > uint64(len(data)) {
|
||
return false
|
||
}
|
||
size := uint64(binary.LittleEndian.Uint16(data[offset+20 : offset+22]))
|
||
return offset+24+size > headerReadLimit
|
||
}
|
||
|
||
func normalizePEData(data []byte) []byte {
|
||
if len(data) < 64 || string(data[:2]) != "MZ" {
|
||
return data
|
||
}
|
||
offset := uint64(binary.LittleEndian.Uint32(data[60:64]))
|
||
if offset < 64 || !peOutsideWindow(data, offset) || offset >= uint64(len(data)) {
|
||
return data
|
||
}
|
||
return normalizePEWindow(data, data[offset:], uint64(len(data))-offset)
|
||
}
|
||
|
||
// A normal CPython cache name and an adjacent regular source file are weaker
|
||
// evidence than orphaned bytecode. Symlinks are not followed for this check.
|
||
func sourceBackedBytecode(path string) bool {
|
||
name := filepath.Base(path)
|
||
dir := filepath.Dir(path)
|
||
var source string
|
||
if filepath.Base(dir) == "__pycache__" {
|
||
i := strings.Index(name, ".cpython-")
|
||
if i <= 0 || !strings.HasSuffix(name, ".pyc") {
|
||
return false
|
||
}
|
||
source = filepath.Join(filepath.Dir(dir), name[:i]+".py")
|
||
} else if strings.HasSuffix(name, ".pyc") {
|
||
source = strings.TrimSuffix(path, ".pyc") + ".py"
|
||
} else {
|
||
return false
|
||
}
|
||
fi, err := os.Lstat(source)
|
||
return err == nil && fi.Mode().IsRegular()
|
||
}
|
||
|
||
func pythonLikePrefix(data []byte) bool {
|
||
if len(data) < 4 || data[2] != '\r' || data[3] != '\n' {
|
||
return false
|
||
}
|
||
if _, known := pythonMagicVersions[binary.LittleEndian.Uint16(data[:2])]; known {
|
||
return true
|
||
}
|
||
// Whitespace followed by CRLF is ordinary text, not bytecode evidence.
|
||
return bytes.IndexFunc(data[:2], func(r rune) bool {
|
||
return r == utf8.RuneError || (unicode.IsControl(r) && !unicode.IsSpace(r))
|
||
}) >= 0
|
||
}
|
||
|
||
// classifyHeader returns at most one finding, plus whether the format is
|
||
// opaque binary content. The caller keeps that content out of text analysis.
|
||
func classifyHeader(data []byte, path string) ([]Finding, bool) {
|
||
if len(data) > headerReadLimit {
|
||
data = data[:headerReadLimit]
|
||
}
|
||
kind, uncertain, archive, script, bytecode := "", false, false, false, false
|
||
switch {
|
||
case bytes.HasPrefix(data, []byte("\x7fELF")):
|
||
kind = "ELF"
|
||
uncertain = !validELFHeader(data)
|
||
case len(data) >= 4 && (string(data[:4]) == "\xfe\xed\xfa\xce" || string(data[:4]) == "\xce\xfa\xed\xfe" || string(data[:4]) == "\xfe\xed\xfa\xcf" || string(data[:4]) == "\xcf\xfa\xed\xfe"):
|
||
kind = "Mach-O"
|
||
var order binary.ByteOrder = binary.BigEndian
|
||
if data[0] == 0xce || data[0] == 0xcf {
|
||
order = binary.LittleEndian
|
||
}
|
||
minSize := 28
|
||
if data[0] == 0xcf || data[3] == 0xcf {
|
||
minSize = 32
|
||
}
|
||
uncertain = len(data) < minSize
|
||
if !uncertain {
|
||
fileType := order.Uint32(data[12:16])
|
||
uncertain = order.Uint32(data[4:8]) == 0 || fileType < 1 || fileType > 12
|
||
}
|
||
case len(data) >= 4 && (string(data[:4]) == "\xca\xfe\xba\xbe" || string(data[:4]) == "\xbe\xba\xfe\xca" || string(data[:4]) == "\xca\xfe\xba\xbf" || string(data[:4]) == "\xbf\xba\xfe\xca"):
|
||
kind = "Mach-O universal"
|
||
if len(data) < 8 {
|
||
uncertain = true
|
||
break
|
||
}
|
||
var order binary.ByteOrder = binary.BigEndian
|
||
if data[0] == 0xbe || data[0] == 0xbf {
|
||
order = binary.LittleEndian
|
||
}
|
||
count := order.Uint32(data[4:8])
|
||
// Java class files share CAFEBABE but have a version here, not a
|
||
// small architecture count. Do not call those native executables.
|
||
if count > 32 {
|
||
return nil, isBinaryContent(data)
|
||
}
|
||
if count == 0 {
|
||
uncertain = true
|
||
break
|
||
}
|
||
entrySize := 20
|
||
if data[0] == 0xbf || data[3] == 0xbf {
|
||
entrySize = 32
|
||
}
|
||
tableEnd := 8 + uint64(count)*uint64(entrySize)
|
||
uncertain = uint64(len(data)) < tableEnd
|
||
if !uncertain {
|
||
for i := uint64(0); i < uint64(count); i++ {
|
||
entryStart := 8 + i*uint64(entrySize)
|
||
if order.Uint32(data[entryStart:entryStart+4]) == 0 {
|
||
uncertain = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
case bytes.HasPrefix(data, []byte("MZ")):
|
||
if len(data) > 2 && !isBinaryContent(data) {
|
||
return nil, false
|
||
}
|
||
kind, uncertain = "PE/DOS-like", true
|
||
if len(data) >= 64 {
|
||
offset := uint64(binary.LittleEndian.Uint32(data[60:64]))
|
||
if offset >= 64 && offset+26 <= uint64(len(data)) && string(data[offset:offset+4]) == "PE\x00\x00" && binary.LittleEndian.Uint16(data[offset+4:offset+6]) != 0 && binary.LittleEndian.Uint16(data[offset+6:offset+8]) != 0 {
|
||
optionalSize := uint64(binary.LittleEndian.Uint16(data[offset+20 : offset+22]))
|
||
optionalStart := offset + 24
|
||
if optionalSize >= 2 && optionalStart+optionalSize <= uint64(len(data)) {
|
||
var minOptionalSize uint64
|
||
switch binary.LittleEndian.Uint16(data[optionalStart : optionalStart+2]) {
|
||
case 0x010b:
|
||
minOptionalSize = 96
|
||
case 0x020b:
|
||
minOptionalSize = 112
|
||
case 0x0107:
|
||
minOptionalSize = 56
|
||
}
|
||
if minOptionalSize != 0 && optionalSize >= minOptionalSize {
|
||
kind, uncertain = "PE", false
|
||
}
|
||
}
|
||
}
|
||
}
|
||
case bytes.HasPrefix(data, []byte("\x00asm")):
|
||
kind = "WASM"
|
||
uncertain = len(data) < 8 || string(data[4:8]) != "\x01\x00\x00\x00"
|
||
case pythonLikePrefix(data):
|
||
kind = "Python bytecode-like"
|
||
if version, ok := pythonMagicVersions[binary.LittleEndian.Uint16(data[:2])]; ok {
|
||
kind, bytecode = "CPython "+version+" bytecode", true
|
||
}
|
||
if bytecode && binary.LittleEndian.Uint16(data[:2]) == 3379 {
|
||
uncertain = len(data) < 12 // Python 3.6 predates the flags field.
|
||
} else {
|
||
uncertain = !bytecode || len(data) < 16 || binary.LittleEndian.Uint32(data[4:8]) & ^uint32(3) != 0
|
||
}
|
||
case bytes.HasPrefix(data, []byte("PK\x03\x04")) || bytes.HasPrefix(data, []byte("PK\x05\x06")) || bytes.HasPrefix(data, []byte("PK\x07\x08")):
|
||
if !validZIPHeader(data) {
|
||
return nil, isBinaryContent(data)
|
||
}
|
||
kind, archive = "ZIP/JAR archive", true
|
||
case bytes.HasPrefix(data, []byte("#!")):
|
||
line, _, _ := bytes.Cut(data[2:], []byte("\n"))
|
||
if len(bytes.TrimSpace(line)) == 0 || !disguisedPath(path) {
|
||
return nil, isBinaryContent(data)
|
||
}
|
||
kind, script = "shebang script", true
|
||
default:
|
||
return nil, isBinaryContent(data)
|
||
}
|
||
pattern, severity := "executable-artifact", SeverityMedium
|
||
if disguisedPath(path) {
|
||
pattern, severity = "disguised-executable", SeverityHigh
|
||
}
|
||
if bytecode {
|
||
pattern, severity = "python-bytecode", SeverityHigh
|
||
}
|
||
if archive {
|
||
pattern, severity = "archive-artifact", SeverityInfo
|
||
}
|
||
if uncertain {
|
||
pattern, severity = "uncertain-header", SeverityInfo
|
||
}
|
||
message := kind + " header detected; runtime behavior has not been analyzed"
|
||
if archive {
|
||
message = kind + " signature detected; contents have not been inspected"
|
||
}
|
||
if uncertain {
|
||
message = kind + " header is unknown, truncated, invalid or outside the 512-byte window; format is uncertain"
|
||
}
|
||
return []Finding{{Severity: severity, Pattern: pattern, RuleID: pattern,
|
||
Message: message, File: path, Analyzer: AnalyzerStatic, Category: CategoryRisk, Confidence: 1.0}}, isBinaryContent(data) || (!script && !uncertain)
|
||
}
|
||
|
||
// inspectHeader keeps collection accounting independent of analyzer policy.
|
||
func (r *Result) inspectHeader(path, classifyPath, displayPath string, size int64, enabled bool, disabled map[string]bool, content ...[]byte) (bool, error) {
|
||
var data []byte
|
||
if len(content) > 0 {
|
||
data = normalizePEData(content[0])
|
||
} else {
|
||
var err error
|
||
data, err = readHeader(path)
|
||
if err != nil {
|
||
return false, fmt.Errorf("read file header %s: %w", displayPath, err)
|
||
}
|
||
}
|
||
findings, opaque := classifyHeader(data, classifyPath)
|
||
for i := range findings {
|
||
if findings[i].Pattern == "python-bytecode" && sourceBackedBytecode(path) {
|
||
findings[i].Severity = SeverityMedium
|
||
}
|
||
findings[i].File = displayPath
|
||
}
|
||
if enabled {
|
||
r.Findings = append(r.Findings, filterDisabledFindings(findings, disabled)...)
|
||
}
|
||
if opaque {
|
||
r.BinaryFiles++
|
||
}
|
||
if size > maxScanFileSize {
|
||
r.OversizedFiles++
|
||
}
|
||
skipped := opaque || size > maxScanFileSize || !isScannable(filepath.Base(displayPath))
|
||
if skipped {
|
||
r.SkippedFiles++
|
||
}
|
||
return skipped, nil
|
||
}
|