lewis.moe
Monorepo for Tangled
spindle: add environment variables to pipeline workflows
Introduces framework-provided TANGLED_* environment variables
(repo info, ref/SHA, trigger-specific data) that are injected
into workflow steps.
Signed-off-by: Evan Jarrett <evan@evanjarrett.com>
authored by
Evan Jarrett
and committed by
Tangled
cb88daef
ca41f242
+4
-5
spindle/engines/nixery/engine.go
+4
-5
spindle/engines/nixery/engine.go
···
73
73
type addlFields struct {
74
74
image string
75
75
container string
76
-
env map[string]string
77
76
}
78
77
79
78
func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) {
···
103
102
swf.Steps = append(swf.Steps, sstep)
104
103
}
105
104
swf.Name = twf.Name
106
-
addl.env = dwf.Environment
105
+
swf.Environment = dwf.Environment
107
106
addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery)
108
107
109
108
setup := &setupSteps{}
···
288
287
289
288
func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *models.WorkflowLogger) error {
290
289
addl := w.Data.(addlFields)
291
-
workflowEnvs := ConstructEnvs(addl.env)
290
+
workflowEnvs := ConstructEnvs(w.Environment)
292
291
// TODO(winter): should SetupWorkflow also have secret access?
293
292
// IMO yes, but probably worth thinking on.
294
293
for _, s := range secrets {
···
310
309
envs.AddEnv("HOME", homeDir)
311
310
312
311
mkExecResp, err := e.docker.ContainerExecCreate(ctx, addl.container, container.ExecOptions{
313
-
Cmd: []string{"bash", "-c", step.command},
312
+
Cmd: []string{"bash", "-c", step.Command()},
314
313
AttachStdout: true,
315
314
AttachStderr: true,
316
315
Env: envs,
···
333
332
// Docker doesn't provide an API to kill an exec run
334
333
// (sure, we could grab the PID and kill it ourselves,
335
334
// but that's wasted effort)
336
-
e.l.Warn("step timed out", "step", step.Name)
335
+
e.l.Warn("step timed out", "step", step.Name())
337
336
338
337
<-tailDone
339
338
+6
-7
spindle/models/clone.go
+6
-7
spindle/models/clone.go
···
55
55
}
56
56
}
57
57
58
-
repoURL := buildRepoURL(tr, dev)
58
+
repoURL := BuildRepoURL(tr.Repo, dev)
59
59
60
60
var cloneOpts tangled.Pipeline_CloneOpts
61
61
if twf.Clone != nil {
···
101
101
}
102
102
}
103
103
104
-
// buildRepoURL constructs the repository URL from trigger metadata
105
-
func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string {
106
-
if tr.Repo == nil {
104
+
// BuildRepoURL constructs the repository URL from repo metadata.
105
+
func BuildRepoURL(repo *tangled.Pipeline_TriggerRepo, devMode bool) string {
106
+
if repo == nil {
107
107
return ""
108
108
}
109
109
110
-
// Determine protocol
111
110
scheme := "https://"
112
111
if devMode {
113
112
scheme = "http://"
114
113
}
115
114
116
115
// Get host from knot
117
-
host := tr.Repo.Knot
116
+
host := repo.Knot
118
117
119
118
// In dev mode, replace localhost with host.docker.internal for Docker networking
120
119
if devMode && strings.Contains(host, "localhost") {
···
122
121
}
123
122
124
123
// Build URL: {scheme}{knot}/{did}/{repo}
125
-
return fmt.Sprintf("%s%s/%s/%s", scheme, host, tr.Repo.Did, tr.Repo.Repo)
124
+
return fmt.Sprintf("%s%s/%s/%s", scheme, host, repo.Did, repo.Repo)
126
125
}
127
126
128
127
// buildFetchArgs constructs the arguments for git fetch based on clone options
+4
-3
spindle/models/pipeline.go
+4
-3
spindle/models/pipeline.go
+77
spindle/models/pipeline_env.go
+77
spindle/models/pipeline_env.go
···
1
+
package models
2
+
3
+
import (
4
+
"strings"
5
+
6
+
"github.com/go-git/go-git/v5/plumbing"
7
+
"tangled.org/core/api/tangled"
8
+
"tangled.org/core/workflow"
9
+
)
10
+
11
+
// PipelineEnvVars extracts environment variables from pipeline trigger metadata.
12
+
// These are framework-provided variables that are injected into workflow steps.
13
+
func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId, devMode bool) map[string]string {
14
+
if tr == nil {
15
+
return nil
16
+
}
17
+
18
+
env := make(map[string]string)
19
+
20
+
// Standard CI environment variable
21
+
env["CI"] = "true"
22
+
23
+
env["TANGLED_PIPELINE_ID"] = pipelineId.Rkey
24
+
25
+
// Repo info
26
+
if tr.Repo != nil {
27
+
env["TANGLED_REPO_KNOT"] = tr.Repo.Knot
28
+
env["TANGLED_REPO_DID"] = tr.Repo.Did
29
+
env["TANGLED_REPO_NAME"] = tr.Repo.Repo
30
+
env["TANGLED_REPO_DEFAULT_BRANCH"] = tr.Repo.DefaultBranch
31
+
env["TANGLED_REPO_URL"] = BuildRepoURL(tr.Repo, devMode)
32
+
}
33
+
34
+
switch workflow.TriggerKind(tr.Kind) {
35
+
case workflow.TriggerKindPush:
36
+
if tr.Push != nil {
37
+
refName := plumbing.ReferenceName(tr.Push.Ref)
38
+
refType := "branch"
39
+
if refName.IsTag() {
40
+
refType = "tag"
41
+
}
42
+
43
+
env["TANGLED_REF"] = tr.Push.Ref
44
+
env["TANGLED_REF_NAME"] = refName.Short()
45
+
env["TANGLED_REF_TYPE"] = refType
46
+
env["TANGLED_SHA"] = tr.Push.NewSha
47
+
env["TANGLED_COMMIT_SHA"] = tr.Push.NewSha
48
+
}
49
+
50
+
case workflow.TriggerKindPullRequest:
51
+
if tr.PullRequest != nil {
52
+
// For PRs, the "ref" is the source branch
53
+
env["TANGLED_REF"] = "refs/heads/" + tr.PullRequest.SourceBranch
54
+
env["TANGLED_REF_NAME"] = tr.PullRequest.SourceBranch
55
+
env["TANGLED_REF_TYPE"] = "branch"
56
+
env["TANGLED_SHA"] = tr.PullRequest.SourceSha
57
+
env["TANGLED_COMMIT_SHA"] = tr.PullRequest.SourceSha
58
+
59
+
// PR-specific variables
60
+
env["TANGLED_PR_SOURCE_BRANCH"] = tr.PullRequest.SourceBranch
61
+
env["TANGLED_PR_TARGET_BRANCH"] = tr.PullRequest.TargetBranch
62
+
env["TANGLED_PR_SOURCE_SHA"] = tr.PullRequest.SourceSha
63
+
env["TANGLED_PR_ACTION"] = tr.PullRequest.Action
64
+
}
65
+
66
+
case workflow.TriggerKindManual:
67
+
// Manual triggers may not have ref/sha info
68
+
// Include any manual inputs if present
69
+
if tr.Manual != nil {
70
+
for _, pair := range tr.Manual.Inputs {
71
+
env["TANGLED_INPUT_"+strings.ToUpper(pair.Key)] = pair.Value
72
+
}
73
+
}
74
+
}
75
+
76
+
return env
77
+
}
+260
spindle/models/pipeline_env_test.go
+260
spindle/models/pipeline_env_test.go
···
1
+
package models
2
+
3
+
import (
4
+
"testing"
5
+
6
+
"tangled.org/core/api/tangled"
7
+
"tangled.org/core/workflow"
8
+
)
9
+
10
+
func TestPipelineEnvVars_PushBranch(t *testing.T) {
11
+
tr := &tangled.Pipeline_TriggerMetadata{
12
+
Kind: string(workflow.TriggerKindPush),
13
+
Push: &tangled.Pipeline_PushTriggerData{
14
+
NewSha: "abc123def456",
15
+
OldSha: "000000000000",
16
+
Ref: "refs/heads/main",
17
+
},
18
+
Repo: &tangled.Pipeline_TriggerRepo{
19
+
Knot: "example.com",
20
+
Did: "did:plc:user123",
21
+
Repo: "my-repo",
22
+
DefaultBranch: "main",
23
+
},
24
+
}
25
+
id := PipelineId{
26
+
Knot: "example.com",
27
+
Rkey: "123123",
28
+
}
29
+
env := PipelineEnvVars(tr, id, false)
30
+
31
+
// Check standard CI variable
32
+
if env["CI"] != "true" {
33
+
t.Errorf("Expected CI='true', got '%s'", env["CI"])
34
+
}
35
+
36
+
// Check ref variables
37
+
if env["TANGLED_REF"] != "refs/heads/main" {
38
+
t.Errorf("Expected TANGLED_REF='refs/heads/main', got '%s'", env["TANGLED_REF"])
39
+
}
40
+
if env["TANGLED_REF_NAME"] != "main" {
41
+
t.Errorf("Expected TANGLED_REF_NAME='main', got '%s'", env["TANGLED_REF_NAME"])
42
+
}
43
+
if env["TANGLED_REF_TYPE"] != "branch" {
44
+
t.Errorf("Expected TANGLED_REF_TYPE='branch', got '%s'", env["TANGLED_REF_TYPE"])
45
+
}
46
+
47
+
// Check SHA variables
48
+
if env["TANGLED_SHA"] != "abc123def456" {
49
+
t.Errorf("Expected TANGLED_SHA='abc123def456', got '%s'", env["TANGLED_SHA"])
50
+
}
51
+
if env["TANGLED_COMMIT_SHA"] != "abc123def456" {
52
+
t.Errorf("Expected TANGLED_COMMIT_SHA='abc123def456', got '%s'", env["TANGLED_COMMIT_SHA"])
53
+
}
54
+
55
+
// Check repo variables
56
+
if env["TANGLED_REPO_KNOT"] != "example.com" {
57
+
t.Errorf("Expected TANGLED_REPO_KNOT='example.com', got '%s'", env["TANGLED_REPO_KNOT"])
58
+
}
59
+
if env["TANGLED_REPO_DID"] != "did:plc:user123" {
60
+
t.Errorf("Expected TANGLED_REPO_DID='did:plc:user123', got '%s'", env["TANGLED_REPO_DID"])
61
+
}
62
+
if env["TANGLED_REPO_NAME"] != "my-repo" {
63
+
t.Errorf("Expected TANGLED_REPO_NAME='my-repo', got '%s'", env["TANGLED_REPO_NAME"])
64
+
}
65
+
if env["TANGLED_REPO_DEFAULT_BRANCH"] != "main" {
66
+
t.Errorf("Expected TANGLED_REPO_DEFAULT_BRANCH='main', got '%s'", env["TANGLED_REPO_DEFAULT_BRANCH"])
67
+
}
68
+
if env["TANGLED_REPO_URL"] != "https://example.com/did:plc:user123/my-repo" {
69
+
t.Errorf("Expected TANGLED_REPO_URL='https://example.com/did:plc:user123/my-repo', got '%s'", env["TANGLED_REPO_URL"])
70
+
}
71
+
}
72
+
73
+
func TestPipelineEnvVars_PushTag(t *testing.T) {
74
+
tr := &tangled.Pipeline_TriggerMetadata{
75
+
Kind: string(workflow.TriggerKindPush),
76
+
Push: &tangled.Pipeline_PushTriggerData{
77
+
NewSha: "abc123def456",
78
+
OldSha: "000000000000",
79
+
Ref: "refs/tags/v1.2.3",
80
+
},
81
+
Repo: &tangled.Pipeline_TriggerRepo{
82
+
Knot: "example.com",
83
+
Did: "did:plc:user123",
84
+
Repo: "my-repo",
85
+
},
86
+
}
87
+
id := PipelineId{
88
+
Knot: "example.com",
89
+
Rkey: "123123",
90
+
}
91
+
env := PipelineEnvVars(tr, id, false)
92
+
93
+
if env["TANGLED_REF"] != "refs/tags/v1.2.3" {
94
+
t.Errorf("Expected TANGLED_REF='refs/tags/v1.2.3', got '%s'", env["TANGLED_REF"])
95
+
}
96
+
if env["TANGLED_REF_NAME"] != "v1.2.3" {
97
+
t.Errorf("Expected TANGLED_REF_NAME='v1.2.3', got '%s'", env["TANGLED_REF_NAME"])
98
+
}
99
+
if env["TANGLED_REF_TYPE"] != "tag" {
100
+
t.Errorf("Expected TANGLED_REF_TYPE='tag', got '%s'", env["TANGLED_REF_TYPE"])
101
+
}
102
+
}
103
+
104
+
func TestPipelineEnvVars_PullRequest(t *testing.T) {
105
+
tr := &tangled.Pipeline_TriggerMetadata{
106
+
Kind: string(workflow.TriggerKindPullRequest),
107
+
PullRequest: &tangled.Pipeline_PullRequestTriggerData{
108
+
SourceBranch: "feature-branch",
109
+
TargetBranch: "main",
110
+
SourceSha: "pr-sha-789",
111
+
Action: "opened",
112
+
},
113
+
Repo: &tangled.Pipeline_TriggerRepo{
114
+
Knot: "example.com",
115
+
Did: "did:plc:user123",
116
+
Repo: "my-repo",
117
+
},
118
+
}
119
+
id := PipelineId{
120
+
Knot: "example.com",
121
+
Rkey: "123123",
122
+
}
123
+
env := PipelineEnvVars(tr, id, false)
124
+
125
+
// Check ref variables for PR
126
+
if env["TANGLED_REF"] != "refs/heads/feature-branch" {
127
+
t.Errorf("Expected TANGLED_REF='refs/heads/feature-branch', got '%s'", env["TANGLED_REF"])
128
+
}
129
+
if env["TANGLED_REF_NAME"] != "feature-branch" {
130
+
t.Errorf("Expected TANGLED_REF_NAME='feature-branch', got '%s'", env["TANGLED_REF_NAME"])
131
+
}
132
+
if env["TANGLED_REF_TYPE"] != "branch" {
133
+
t.Errorf("Expected TANGLED_REF_TYPE='branch', got '%s'", env["TANGLED_REF_TYPE"])
134
+
}
135
+
136
+
// Check SHA variables
137
+
if env["TANGLED_SHA"] != "pr-sha-789" {
138
+
t.Errorf("Expected TANGLED_SHA='pr-sha-789', got '%s'", env["TANGLED_SHA"])
139
+
}
140
+
if env["TANGLED_COMMIT_SHA"] != "pr-sha-789" {
141
+
t.Errorf("Expected TANGLED_COMMIT_SHA='pr-sha-789', got '%s'", env["TANGLED_COMMIT_SHA"])
142
+
}
143
+
144
+
// Check PR-specific variables
145
+
if env["TANGLED_PR_SOURCE_BRANCH"] != "feature-branch" {
146
+
t.Errorf("Expected TANGLED_PR_SOURCE_BRANCH='feature-branch', got '%s'", env["TANGLED_PR_SOURCE_BRANCH"])
147
+
}
148
+
if env["TANGLED_PR_TARGET_BRANCH"] != "main" {
149
+
t.Errorf("Expected TANGLED_PR_TARGET_BRANCH='main', got '%s'", env["TANGLED_PR_TARGET_BRANCH"])
150
+
}
151
+
if env["TANGLED_PR_SOURCE_SHA"] != "pr-sha-789" {
152
+
t.Errorf("Expected TANGLED_PR_SOURCE_SHA='pr-sha-789', got '%s'", env["TANGLED_PR_SOURCE_SHA"])
153
+
}
154
+
if env["TANGLED_PR_ACTION"] != "opened" {
155
+
t.Errorf("Expected TANGLED_PR_ACTION='opened', got '%s'", env["TANGLED_PR_ACTION"])
156
+
}
157
+
}
158
+
159
+
func TestPipelineEnvVars_ManualWithInputs(t *testing.T) {
160
+
tr := &tangled.Pipeline_TriggerMetadata{
161
+
Kind: string(workflow.TriggerKindManual),
162
+
Manual: &tangled.Pipeline_ManualTriggerData{
163
+
Inputs: []*tangled.Pipeline_Pair{
164
+
{Key: "version", Value: "1.0.0"},
165
+
{Key: "environment", Value: "production"},
166
+
},
167
+
},
168
+
Repo: &tangled.Pipeline_TriggerRepo{
169
+
Knot: "example.com",
170
+
Did: "did:plc:user123",
171
+
Repo: "my-repo",
172
+
},
173
+
}
174
+
id := PipelineId{
175
+
Knot: "example.com",
176
+
Rkey: "123123",
177
+
}
178
+
env := PipelineEnvVars(tr, id, false)
179
+
180
+
// Check manual input variables
181
+
if env["TANGLED_INPUT_VERSION"] != "1.0.0" {
182
+
t.Errorf("Expected TANGLED_INPUT_VERSION='1.0.0', got '%s'", env["TANGLED_INPUT_VERSION"])
183
+
}
184
+
if env["TANGLED_INPUT_ENVIRONMENT"] != "production" {
185
+
t.Errorf("Expected TANGLED_INPUT_ENVIRONMENT='production', got '%s'", env["TANGLED_INPUT_ENVIRONMENT"])
186
+
}
187
+
188
+
// Manual triggers shouldn't have ref/sha variables
189
+
if _, ok := env["TANGLED_REF"]; ok {
190
+
t.Error("Manual trigger should not have TANGLED_REF")
191
+
}
192
+
if _, ok := env["TANGLED_SHA"]; ok {
193
+
t.Error("Manual trigger should not have TANGLED_SHA")
194
+
}
195
+
}
196
+
197
+
func TestPipelineEnvVars_DevMode(t *testing.T) {
198
+
tr := &tangled.Pipeline_TriggerMetadata{
199
+
Kind: string(workflow.TriggerKindPush),
200
+
Push: &tangled.Pipeline_PushTriggerData{
201
+
NewSha: "abc123",
202
+
Ref: "refs/heads/main",
203
+
},
204
+
Repo: &tangled.Pipeline_TriggerRepo{
205
+
Knot: "localhost:3000",
206
+
Did: "did:plc:user123",
207
+
Repo: "my-repo",
208
+
},
209
+
}
210
+
id := PipelineId{
211
+
Knot: "example.com",
212
+
Rkey: "123123",
213
+
}
214
+
env := PipelineEnvVars(tr, id, true)
215
+
216
+
// Dev mode should use http:// and replace localhost with host.docker.internal
217
+
expectedURL := "http://host.docker.internal:3000/did:plc:user123/my-repo"
218
+
if env["TANGLED_REPO_URL"] != expectedURL {
219
+
t.Errorf("Expected TANGLED_REPO_URL='%s', got '%s'", expectedURL, env["TANGLED_REPO_URL"])
220
+
}
221
+
}
222
+
223
+
func TestPipelineEnvVars_NilTrigger(t *testing.T) {
224
+
id := PipelineId{
225
+
Knot: "example.com",
226
+
Rkey: "123123",
227
+
}
228
+
env := PipelineEnvVars(nil, id, false)
229
+
230
+
if env != nil {
231
+
t.Error("Expected nil env for nil trigger")
232
+
}
233
+
}
234
+
235
+
func TestPipelineEnvVars_NilPushData(t *testing.T) {
236
+
tr := &tangled.Pipeline_TriggerMetadata{
237
+
Kind: string(workflow.TriggerKindPush),
238
+
Push: nil,
239
+
Repo: &tangled.Pipeline_TriggerRepo{
240
+
Knot: "example.com",
241
+
Did: "did:plc:user123",
242
+
Repo: "my-repo",
243
+
},
244
+
}
245
+
id := PipelineId{
246
+
Knot: "example.com",
247
+
Rkey: "123123",
248
+
}
249
+
env := PipelineEnvVars(tr, id, false)
250
+
251
+
// Should still have repo variables
252
+
if env["TANGLED_REPO_KNOT"] != "example.com" {
253
+
t.Errorf("Expected TANGLED_REPO_KNOT='example.com', got '%s'", env["TANGLED_REPO_KNOT"])
254
+
}
255
+
256
+
// Should not have ref/sha variables
257
+
if _, ok := env["TANGLED_REF"]; ok {
258
+
t.Error("Should not have TANGLED_REF when push data is nil")
259
+
}
260
+
}
+11
spindle/server.go
+11
spindle/server.go
···
6
6
"encoding/json"
7
7
"fmt"
8
8
"log/slog"
9
+
"maps"
9
10
"net/http"
10
11
11
12
"github.com/go-chi/chi/v5"
···
311
312
312
313
workflows := make(map[models.Engine][]models.Workflow)
313
314
315
+
// Build pipeline environment variables once for all workflows
316
+
pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId, s.cfg.Server.Dev)
317
+
314
318
for _, w := range tpl.Workflows {
315
319
if w != nil {
316
320
if _, ok := s.engs[w.Engine]; !ok {
···
335
339
if err != nil {
336
340
return err
337
341
}
342
+
343
+
// inject TANGLED_* env vars after InitWorkflow
344
+
// This prevents user-defined env vars from overriding them
345
+
if ewf.Environment == nil {
346
+
ewf.Environment = make(map[string]string)
347
+
}
348
+
maps.Copy(ewf.Environment, pipelineEnv)
338
349
339
350
workflows[eng] = append(workflows[eng], *ewf)
340
351