mirror of
https://github.com/lunchcat/sif.git
synced 2026-07-03 11:24:54 -07:00
feat(modules): detect exposed ml experiment trackers (#243)
add recon modules for self-hosted training and experiment-tracking platforms reachable without auth: mlflow, tensorboard, aim, and determined disclose experiments, the artifact store, training run paths, and cluster topology over unauthenticated apis.
This commit is contained in:
Tigah
@@ -0,0 +1,223 @@
|
||||
package modules_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/vmfunc/sif/internal/modules"
|
||||
)
|
||||
|
||||
func runTrackingModule(t *testing.T, file string, status int, body string) *modules.Result {
|
||||
t.Helper()
|
||||
def, err := modules.ParseYAMLModule(file)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %s: %v", file, err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
res, err := modules.ExecuteHTTPModule(context.Background(), srv.URL, def, modules.Options{
|
||||
Timeout: 5 * time.Second,
|
||||
Threads: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("execute %s: %v", file, err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func trackingExtract(res *modules.Result, key string) string {
|
||||
for _, f := range res.Findings {
|
||||
if v := f.Extracted[key]; v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestMLTrackingExposureModules(t *testing.T) {
|
||||
const mlflow = "../../modules/recon/mlflow-api-exposure.yaml"
|
||||
const tensorboard = "../../modules/recon/tensorboard-exposure.yaml"
|
||||
const aim = "../../modules/recon/aim-exposure.yaml"
|
||||
const determined = "../../modules/recon/determined-master-exposure.yaml"
|
||||
|
||||
mlflowExperiment := `{"experiment":{"experiment_id":"0","name":"Default",` +
|
||||
`"artifact_location":"file:///mlflow/mlruns/0","lifecycle_stage":"active",` +
|
||||
`"creation_time":1700000000000,"last_update_time":1700000000000,"tags":[]}}`
|
||||
|
||||
tensorboardEnv := `{"data_location":"/home/ml/runs/exp-2024","window_title":"",` +
|
||||
`"experiment_name":"","experiment_description":"","creation_time":0,"version":"2.16.2"}`
|
||||
|
||||
aimProject := `{"name":"my-aim-repo","path":"/home/ml/.aim","description":"",` +
|
||||
`"telemetry_enabled":0,"warn_index":false,"warn_runs":false}`
|
||||
|
||||
determinedMaster := `{"version":"0.27.1","master_id":"6f1f2a9c","cluster_id":"a1b2c3d4-e5f6-7890",` +
|
||||
`"cluster_name":"prod-cluster","telemetry_enabled":true,"rbac_enabled":false,` +
|
||||
`"strict_job_queue_control":false,"has_custom_logo":false,"branding":"determined"}`
|
||||
|
||||
t.Run("an mlflow experiment is flagged with its artifact store", func(t *testing.T) {
|
||||
res := runTrackingModule(t, mlflow, 200, mlflowExperiment)
|
||||
if len(res.Findings) == 0 {
|
||||
t.Fatal("expected an mlflow finding")
|
||||
}
|
||||
if v := trackingExtract(res, "mlflow_artifact_location"); v != "file:///mlflow/mlruns/0" {
|
||||
t.Errorf("mlflow_artifact_location=%q, want file:///mlflow/mlruns/0", v)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an experiment_id without lifecycle_stage is not flagged as mlflow", func(t *testing.T) {
|
||||
body := `{"experiment_id":"5","artifact_location":"s3://bucket/x"}`
|
||||
if res := runTrackingModule(t, mlflow, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a partial body should not match mlflow, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an experiment without an artifact_location is not flagged as mlflow", func(t *testing.T) {
|
||||
body := `{"experiment":{"experiment_id":"0","name":"Default","lifecycle_stage":"active"}}`
|
||||
if res := runTrackingModule(t, mlflow, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("an artifactless experiment should not match mlflow, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an experiment without an experiment_id is not flagged as mlflow", func(t *testing.T) {
|
||||
body := `{"experiment":{"name":"Default","artifact_location":"file:///x","lifecycle_stage":"active"}}`
|
||||
if res := runTrackingModule(t, mlflow, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("an idless experiment should not match mlflow, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a generic lifecycle body is not flagged as mlflow", func(t *testing.T) {
|
||||
body := `{"lifecycle_stage":"production","name":"some-service"}`
|
||||
if res := runTrackingModule(t, mlflow, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a generic lifecycle body should not match mlflow, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a model checkpoint list is not flagged as mlflow", func(t *testing.T) {
|
||||
body := `[{"title":"x","model_name":"y","filename":"z"}]`
|
||||
if res := runTrackingModule(t, mlflow, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a model list should not match mlflow, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a tensorboard environment is flagged with its version and run path", func(t *testing.T) {
|
||||
res := runTrackingModule(t, tensorboard, 200, tensorboardEnv)
|
||||
if len(res.Findings) == 0 {
|
||||
t.Fatal("expected a tensorboard finding")
|
||||
}
|
||||
if v := trackingExtract(res, "tensorboard_version"); v != "2.16.2" {
|
||||
t.Errorf("tensorboard_version=%q, want 2.16.2", v)
|
||||
}
|
||||
if v := trackingExtract(res, "tensorboard_data_location"); v != "/home/ml/runs/exp-2024" {
|
||||
t.Errorf("tensorboard_data_location=%q, want /home/ml/runs/exp-2024", v)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a body without data_location is not flagged as tensorboard", func(t *testing.T) {
|
||||
body := `{"window_title":"","version":"2.16.2"}`
|
||||
if res := runTrackingModule(t, tensorboard, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without data_location should not match tensorboard, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a body without window_title is not flagged as tensorboard", func(t *testing.T) {
|
||||
body := `{"data_location":"/runs","version":"2.16.2"}`
|
||||
if res := runTrackingModule(t, tensorboard, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without window_title should not match tensorboard, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a body without a version is not flagged as tensorboard", func(t *testing.T) {
|
||||
body := `{"data_location":"/runs","window_title":"my board"}`
|
||||
if res := runTrackingModule(t, tensorboard, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without version should not match tensorboard, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an aim project is flagged with its repo path", func(t *testing.T) {
|
||||
res := runTrackingModule(t, aim, 200, aimProject)
|
||||
if len(res.Findings) == 0 {
|
||||
t.Fatal("expected an aim finding")
|
||||
}
|
||||
if v := trackingExtract(res, "aim_project_path"); v != "/home/ml/.aim" {
|
||||
t.Errorf("aim_project_path=%q, want /home/ml/.aim", v)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a body without telemetry_enabled is not flagged as aim", func(t *testing.T) {
|
||||
body := `{"name":"x","path":"/y","warn_index":false,"warn_runs":false}`
|
||||
if res := runTrackingModule(t, aim, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without telemetry_enabled should not match aim, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a body without warn_index is not flagged as aim", func(t *testing.T) {
|
||||
body := `{"telemetry_enabled":0,"warn_runs":false,"name":"x"}`
|
||||
if res := runTrackingModule(t, aim, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without warn_index should not match aim, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a body without warn_runs is not flagged as aim", func(t *testing.T) {
|
||||
body := `{"telemetry_enabled":0,"warn_index":false,"name":"x"}`
|
||||
if res := runTrackingModule(t, aim, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without warn_runs should not match aim, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a determined master is flagged with its version and cluster id", func(t *testing.T) {
|
||||
res := runTrackingModule(t, determined, 200, determinedMaster)
|
||||
if len(res.Findings) == 0 {
|
||||
t.Fatal("expected a determined finding")
|
||||
}
|
||||
if v := trackingExtract(res, "determined_version"); v != "0.27.1" {
|
||||
t.Errorf("determined_version=%q, want 0.27.1", v)
|
||||
}
|
||||
if v := trackingExtract(res, "determined_cluster_id"); v != "a1b2c3d4-e5f6-7890" {
|
||||
t.Errorf("determined_cluster_id=%q, want a1b2c3d4-e5f6-7890", v)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a cluster info without a master_id is not flagged as determined", func(t *testing.T) {
|
||||
body := `{"cluster_id":"x","cluster_name":"y","version":"1.0"}`
|
||||
if res := runTrackingModule(t, determined, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a masterless cluster info should not match determined, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a master without a cluster_id is not flagged as determined", func(t *testing.T) {
|
||||
body := `{"master_id":"x","cluster_name":"y","version":"1.0"}`
|
||||
if res := runTrackingModule(t, determined, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without cluster_id should not match determined, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a master without a cluster_name is not flagged as determined", func(t *testing.T) {
|
||||
body := `{"master_id":"x","cluster_id":"y","version":"1.0"}`
|
||||
if res := runTrackingModule(t, determined, 200, body); len(res.Findings) > 0 {
|
||||
t.Errorf("a body without cluster_name should not match determined, got %d findings", len(res.Findings))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a plain 200 body is not a leak", func(t *testing.T) {
|
||||
for _, file := range []string{mlflow, tensorboard, aim, determined} {
|
||||
if res := runTrackingModule(t, file, 200, "ok"); len(res.Findings) > 0 {
|
||||
t.Errorf("%s: a plain 200 body should not match, got %d findings", file, len(res.Findings))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a 404 is not a leak", func(t *testing.T) {
|
||||
for _, file := range []string{mlflow, tensorboard, aim, determined} {
|
||||
if res := runTrackingModule(t, file, 404, "not found"); len(res.Findings) > 0 {
|
||||
t.Errorf("%s: a 404 should not match, got %d findings", file, len(res.Findings))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# Aim Experiment Tracker Exposure Detection Module
|
||||
|
||||
id: aim-exposure
|
||||
info:
|
||||
name: Aim Experiment Tracker Exposure
|
||||
author: sif
|
||||
severity: medium
|
||||
description: Detects an exposed Aim experiment tracker; its unauthenticated project api discloses the tracking repo path and serves all logged metrics, params, images, and texts
|
||||
tags: [aim, ai, ml, mlops, training, exposure, recon]
|
||||
|
||||
type: http
|
||||
|
||||
http:
|
||||
method: GET
|
||||
paths:
|
||||
- "{{BaseURL}}/api/projects/"
|
||||
|
||||
matchers:
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"telemetry_enabled\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"warn_index\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"warn_runs\""
|
||||
|
||||
extractors:
|
||||
- type: regex
|
||||
name: aim_project_path
|
||||
part: body
|
||||
regex:
|
||||
- '"path"\s*:\s*"([^"]+)"'
|
||||
group: 1
|
||||
@@ -0,0 +1,51 @@
|
||||
# Determined AI Master Exposure Detection Module
|
||||
|
||||
id: determined-master-exposure
|
||||
info:
|
||||
name: Determined AI Master Exposure
|
||||
author: sif
|
||||
severity: medium
|
||||
description: Detects an exposed Determined AI master over its unauthenticated info endpoint, disclosing the cluster id, version, and auth configuration; Determined ships a default admin account whose blank password permits job submission and code execution on the cluster
|
||||
tags: [determined, ai, ml, mlops, training, exposure, recon]
|
||||
|
||||
type: http
|
||||
|
||||
http:
|
||||
method: GET
|
||||
paths:
|
||||
- "{{BaseURL}}/api/v1/master"
|
||||
|
||||
matchers:
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"master_id\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"cluster_id\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"cluster_name\""
|
||||
|
||||
extractors:
|
||||
- type: regex
|
||||
name: determined_version
|
||||
part: body
|
||||
regex:
|
||||
- '"version"\s*:\s*"([^"]+)"'
|
||||
group: 1
|
||||
|
||||
- type: regex
|
||||
name: determined_cluster_id
|
||||
part: body
|
||||
regex:
|
||||
- '"cluster_id"\s*:\s*"([^"]+)"'
|
||||
group: 1
|
||||
@@ -0,0 +1,44 @@
|
||||
# MLflow Tracking Server Exposure Detection Module
|
||||
|
||||
id: mlflow-api-exposure
|
||||
info:
|
||||
name: MLflow Tracking Server Exposure
|
||||
author: sif
|
||||
severity: high
|
||||
description: Detects an MLflow tracking server reachable without auth, disclosing experiments and the artifact store over its rest api; exposed servers have historically allowed arbitrary file read through artifact paths
|
||||
tags: [mlflow, ai, ml, mlops, exposure, recon]
|
||||
|
||||
type: http
|
||||
|
||||
http:
|
||||
method: GET
|
||||
paths:
|
||||
- "{{BaseURL}}/api/2.0/mlflow/experiments/get-by-name?experiment_name=Default"
|
||||
|
||||
matchers:
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"artifact_location\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"lifecycle_stage\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"experiment_id\""
|
||||
|
||||
extractors:
|
||||
- type: regex
|
||||
name: mlflow_artifact_location
|
||||
part: body
|
||||
regex:
|
||||
- '"artifact_location"\s*:\s*"([^"]+)"'
|
||||
group: 1
|
||||
@@ -0,0 +1,51 @@
|
||||
# TensorBoard Exposure Detection Module
|
||||
|
||||
id: tensorboard-exposure
|
||||
info:
|
||||
name: TensorBoard Exposure
|
||||
author: sif
|
||||
severity: medium
|
||||
description: Detects an exposed TensorBoard instance; its unauthenticated data api discloses the training run source path and server version, and serves all logged scalars, graphs, images, and embeddings
|
||||
tags: [tensorboard, ai, ml, mlops, training, exposure, recon]
|
||||
|
||||
type: http
|
||||
|
||||
http:
|
||||
method: GET
|
||||
paths:
|
||||
- "{{BaseURL}}/data/environment"
|
||||
|
||||
matchers:
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"data_location\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"window_title\""
|
||||
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "\"version\""
|
||||
|
||||
extractors:
|
||||
- type: regex
|
||||
name: tensorboard_version
|
||||
part: body
|
||||
regex:
|
||||
- '"version"\s*:\s*"([^"]+)"'
|
||||
group: 1
|
||||
|
||||
- type: regex
|
||||
name: tensorboard_data_location
|
||||
part: body
|
||||
regex:
|
||||
- '"data_location"\s*:\s*"([^"]+)"'
|
||||
group: 1
|
||||
Reference in New Issue
Block a user