Skip to content

Commit 1546d97

Browse files
aktechaktech
committed
test(e2e): bundle import via API round-trips assets in local mode
Adds TestE2E_BundlePublishImportViaAPI_LocalMode which seeds an in-memory OCI registry via oci.Publish, imports via POST /registries/:id/import, polls the workspace to ready, and asserts notebook.ipynb and pixi.lock land on disk with correct content. Two supporting fixes uncovered by the test: - config: add explicit BindEnv calls so nested env vars like NEBI_PACKAGE_MANAGER_PIXI_PATH and NEBI_STORAGE_WORKSPACES_DIR are propagated through viper Unmarshal (AutomaticEnv alone does not suffice for nested structs). - router: skip rbac.InitEnforcer in local mode to prevent the local-mode server from clobbering the global casbin enforcer that was already initialised by a concurrently-running team-mode server.
1 parent 21297ba commit 1546d97

3 files changed

Lines changed: 322 additions & 4 deletions

File tree

‎cmd/nebi/bundle_api_e2e_test.go‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
//go:build e2e
2+
3+
package main
4+
5+
import (
6+
"bytes"
7+
"context"
8+
"encoding/json"
9+
"fmt"
10+
"io"
11+
"net/http"
12+
"net/http/httptest"
13+
"net/url"
14+
"os"
15+
"path/filepath"
16+
"testing"
17+
"time"
18+
19+
"github.com/google/go-containerregistry/pkg/registry"
20+
"github.com/nebari-dev/nebi/internal/cliclient"
21+
"github.com/nebari-dev/nebi/internal/oci"
22+
"github.com/nebari-dev/nebi/internal/server"
23+
)
24+
25+
// TestE2E_BundlePublishImportViaAPI_LocalMode verifies that importing a bundle
26+
// via the Nebi HTTP API in local mode correctly extracts all asset layers to
27+
// the workspace directory on disk. It:
28+
//
29+
// 1. Starts a fresh Nebi server in local mode (independent of the shared TestMain server).
30+
// 2. Seeds an in-memory OCI registry with a bundle containing pixi.toml,
31+
// pixi.lock, and notebook.ipynb using oci.Publish directly.
32+
// 3. Creates the registry in Nebi via POST /admin/registries.
33+
// 4. Imports via POST /registries/:id/import.
34+
// 5. Polls GET /workspaces/:id until status == "ready".
35+
// 6. Asserts notebook.ipynb and pixi.lock exist on disk with correct content.
36+
func TestE2E_BundlePublishImportViaAPI_LocalMode(t *testing.T) {
37+
// Snapshot and restore env vars that overlap with the global TestMain setup.
38+
// We need to override them for this test's private server without poisoning
39+
// the shared server that TestMain already started on a different port.
40+
envVars := []string{
41+
"NEBI_MODE",
42+
"NEBI_DATABASE_DSN",
43+
"NEBI_STORAGE_WORKSPACES_DIR",
44+
"NEBI_SERVER_PORT",
45+
"NEBI_PACKAGE_MANAGER_PIXI_PATH",
46+
}
47+
saved := make(map[string]string, len(envVars))
48+
for _, k := range envVars {
49+
saved[k] = os.Getenv(k)
50+
}
51+
t.Cleanup(func() {
52+
for k, v := range saved {
53+
if v == "" {
54+
os.Unsetenv(k)
55+
} else {
56+
os.Setenv(k, v)
57+
}
58+
}
59+
})
60+
61+
// ---- Temp directories ----
62+
dbDir := t.TempDir()
63+
dbPath := filepath.Join(dbDir, "bundle-api-e2e.db")
64+
wsDir := t.TempDir()
65+
66+
// ---- Port allocation ----
67+
port, err := findFreePort()
68+
if err != nil {
69+
t.Fatalf("find free port: %v", err)
70+
}
71+
72+
// ---- Set env vars for local-mode server ----
73+
os.Setenv("NEBI_MODE", "local")
74+
os.Setenv("NEBI_DATABASE_DSN", dbPath)
75+
os.Setenv("NEBI_STORAGE_WORKSPACES_DIR", wsDir)
76+
os.Setenv("NEBI_SERVER_PORT", fmt.Sprintf("%d", port))
77+
// Point pixi to a no-op binary so "pixi install" succeeds without
78+
// actually installing any packages. The workspace only needs files on
79+
// disk for this test to pass. Use /usr/bin/true (macOS) with /bin/true
80+
// as a fallback for Linux.
81+
noopBinary := "/usr/bin/true"
82+
if _, err := os.Stat(noopBinary); err != nil {
83+
noopBinary = "/bin/true"
84+
}
85+
os.Setenv("NEBI_PACKAGE_MANAGER_PIXI_PATH", noopBinary)
86+
87+
serverURL := fmt.Sprintf("http://127.0.0.1:%d", port)
88+
89+
// ---- Start local-mode server ----
90+
ctx, cancel := context.WithCancel(context.Background())
91+
t.Cleanup(cancel)
92+
93+
serverErr := make(chan error, 1)
94+
go func() {
95+
serverErr <- server.Run(ctx, server.Config{
96+
Port: port,
97+
Mode: "both",
98+
Version: "e2e-bundle-api-test",
99+
})
100+
}()
101+
102+
waitForHealth(serverURL+"/api/v1/health", serverErr, io.Discard)
103+
104+
// ---- Login ----
105+
unauthClient := cliclient.NewWithoutAuth(serverURL)
106+
loginResp, err := unauthClient.Login(ctx, "admin", "adminpass")
107+
if err != nil {
108+
t.Fatalf("login: %v", err)
109+
}
110+
token := loginResp.Token
111+
112+
// ---- In-memory OCI registry ----
113+
ociSrv := httptest.NewServer(registry.New())
114+
t.Cleanup(ociSrv.Close)
115+
ociURL, _ := url.Parse(ociSrv.URL)
116+
ociHost := ociURL.Host
117+
118+
// ---- Seed the OCI registry via oci.Publish ----
119+
const (
120+
repoName = "notebook-env"
121+
bundleTag = "v1"
122+
ociNS = "demo"
123+
notebookBody = `{"cells":[],"metadata":{},"nbformat":4,"nbformat_minor":5}`
124+
pixiTomlBody = "[project]\nname = \"notebook-env\"\nchannels = [\"conda-forge\"]\nplatforms = [\"linux-64\"]\n"
125+
pixiLockBody = "version: 6\n# seeded-by-test\n"
126+
)
127+
128+
srcDir := t.TempDir()
129+
writeFile := func(rel, body string) {
130+
t.Helper()
131+
if err := os.MkdirAll(filepath.Dir(filepath.Join(srcDir, rel)), 0o755); err != nil {
132+
t.Fatalf("mkdir for %s: %v", rel, err)
133+
}
134+
if err := os.WriteFile(filepath.Join(srcDir, rel), []byte(body), 0o644); err != nil {
135+
t.Fatalf("write %s: %v", rel, err)
136+
}
137+
}
138+
writeFile("pixi.toml", pixiTomlBody)
139+
writeFile("pixi.lock", pixiLockBody)
140+
writeFile("notebook.ipynb", notebookBody)
141+
142+
reg := oci.Registry{Host: ociHost, Namespace: ociNS, PlainHTTP: true}
143+
if _, err := oci.Publish(ctx, srcDir, reg, repoName, bundleTag); err != nil {
144+
t.Fatalf("seed oci.Publish: %v", err)
145+
}
146+
147+
// ---- Create registry in Nebi via POST /admin/registries ----
148+
registryID := createRegistryViaAPI(t, serverURL, token, map[string]interface{}{
149+
"name": "bundle-api-e2e-reg",
150+
"url": "http://" + ociHost,
151+
"namespace": ociNS,
152+
"is_default": true,
153+
})
154+
155+
// ---- Import via POST /registries/:id/import ----
156+
wsID := importViaAPI(t, serverURL, token, registryID, map[string]interface{}{
157+
"repository": repoName,
158+
"tag": bundleTag,
159+
"name": "notebook-imported",
160+
})
161+
162+
// ---- Poll workspace until ready (or fail after 30s) ----
163+
pollWorkspaceReady(t, serverURL, token, wsID, 30*time.Second)
164+
165+
// ---- Derive workspace directory path ----
166+
// LocalExecutor.GetWorkspacePath: {wsDir}/{normalized-name}-{uuid}
167+
// normalized("notebook-imported") → "notebook-imported"
168+
wsPath := filepath.Join(wsDir, fmt.Sprintf("notebook-imported-%s", wsID))
169+
170+
// ---- Assertions ----
171+
assertFileContent(t, wsPath, "notebook.ipynb", notebookBody)
172+
assertFileContent(t, wsPath, "pixi.lock", pixiLockBody)
173+
assertFileContains(t, wsPath, "pixi.toml", "notebook-env")
174+
}
175+
176+
// createRegistryViaAPI POSTs to /admin/registries and returns the created registry ID.
177+
func createRegistryViaAPI(t *testing.T, serverURL, token string, body map[string]interface{}) string {
178+
t.Helper()
179+
b, _ := json.Marshal(body)
180+
req, _ := http.NewRequest(http.MethodPost, serverURL+"/api/v1/admin/registries", bytes.NewReader(b))
181+
req.Header.Set("Authorization", "Bearer "+token)
182+
req.Header.Set("Content-Type", "application/json")
183+
resp, err := http.DefaultClient.Do(req)
184+
if err != nil {
185+
t.Fatalf("POST /admin/registries: %v", err)
186+
}
187+
defer resp.Body.Close()
188+
raw, _ := io.ReadAll(resp.Body)
189+
if resp.StatusCode != http.StatusCreated {
190+
t.Fatalf("POST /admin/registries: status %d, body: %s", resp.StatusCode, raw)
191+
}
192+
var result struct {
193+
ID string `json:"id"`
194+
}
195+
if err := json.Unmarshal(raw, &result); err != nil {
196+
t.Fatalf("decode registry response: %v (body: %s)", err, raw)
197+
}
198+
if result.ID == "" {
199+
t.Fatalf("registry response missing id: %s", raw)
200+
}
201+
return result.ID
202+
}
203+
204+
// importViaAPI POSTs to /registries/:id/import and returns the created workspace ID.
205+
func importViaAPI(t *testing.T, serverURL, token, registryID string, body map[string]interface{}) string {
206+
t.Helper()
207+
b, _ := json.Marshal(body)
208+
url := fmt.Sprintf("%s/api/v1/registries/%s/import", serverURL, registryID)
209+
req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader(b))
210+
req.Header.Set("Authorization", "Bearer "+token)
211+
req.Header.Set("Content-Type", "application/json")
212+
resp, err := http.DefaultClient.Do(req)
213+
if err != nil {
214+
t.Fatalf("POST /registries/%s/import: %v", registryID, err)
215+
}
216+
defer resp.Body.Close()
217+
raw, _ := io.ReadAll(resp.Body)
218+
if resp.StatusCode != http.StatusCreated {
219+
t.Fatalf("POST /registries/%s/import: status %d, body: %s", registryID, resp.StatusCode, raw)
220+
}
221+
var result struct {
222+
ID string `json:"id"`
223+
}
224+
if err := json.Unmarshal(raw, &result); err != nil {
225+
t.Fatalf("decode import response: %v (body: %s)", err, raw)
226+
}
227+
if result.ID == "" {
228+
t.Fatalf("import response missing id: %s", raw)
229+
}
230+
return result.ID
231+
}
232+
233+
// pollWorkspaceReady polls GET /workspaces/:id until status == "ready" or times out.
234+
func pollWorkspaceReady(t *testing.T, serverURL, token, wsID string, timeout time.Duration) {
235+
t.Helper()
236+
deadline := time.Now().Add(timeout)
237+
url := fmt.Sprintf("%s/api/v1/workspaces/%s", serverURL, wsID)
238+
for time.Now().Before(deadline) {
239+
req, _ := http.NewRequest(http.MethodGet, url, nil)
240+
req.Header.Set("Authorization", "Bearer "+token)
241+
resp, err := http.DefaultClient.Do(req)
242+
if err != nil {
243+
time.Sleep(250 * time.Millisecond)
244+
continue
245+
}
246+
raw, _ := io.ReadAll(resp.Body)
247+
resp.Body.Close()
248+
var ws struct {
249+
Status string `json:"status"`
250+
}
251+
if err := json.Unmarshal(raw, &ws); err != nil {
252+
t.Fatalf("decode workspace response: %v (body: %s)", err, raw)
253+
}
254+
switch ws.Status {
255+
case "ready":
256+
return
257+
case "failed":
258+
t.Fatalf("workspace %s entered 'failed' state", wsID)
259+
}
260+
time.Sleep(250 * time.Millisecond)
261+
}
262+
t.Fatalf("workspace %s did not become ready within %s", wsID, timeout)
263+
}
264+
265+
// assertFileContent reads rel inside wsDir and checks it equals want exactly.
266+
func assertFileContent(t *testing.T, wsDir, rel, want string) {
267+
t.Helper()
268+
got, err := os.ReadFile(filepath.Join(wsDir, rel))
269+
if err != nil {
270+
t.Fatalf("read %s: %v", rel, err)
271+
}
272+
if string(got) != want {
273+
t.Fatalf("content mismatch for %s:\n got: %q\nwant: %q", rel, got, want)
274+
}
275+
}
276+
277+
// assertFileContains reads rel inside wsDir and checks it contains substr.
278+
func assertFileContains(t *testing.T, wsDir, rel, substr string) {
279+
t.Helper()
280+
got, err := os.ReadFile(filepath.Join(wsDir, rel))
281+
if err != nil {
282+
t.Fatalf("read %s: %v", rel, err)
283+
}
284+
if !bytes.Contains(got, []byte(substr)) {
285+
t.Fatalf("%s does not contain %q:\n%s", rel, substr, got)
286+
}
287+
}

‎internal/api/router.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,17 @@ import (
2727

2828
// NewRouter creates and configures the Gin router
2929
func NewRouter(cfg *config.Config, db *gorm.DB, q queue.Queue, exec executor.Executor, logBroker *logstream.LogBroker, valkeyClient interface{}, logger *slog.Logger) *gin.Engine {
30-
// Initialize RBAC enforcer and provider
31-
if err := rbac.InitEnforcer(db, logger); err != nil {
32-
logger.Error("Failed to initialize RBAC", "error", err)
33-
panic(err)
30+
// Initialize RBAC enforcer and provider.
31+
// In local mode the admin and workspace RBAC checks are unconditionally
32+
// skipped (see RequireAdmin / RequireWorkspaceAccess middleware), so
33+
// there is no need to re-initialise the global casbin enforcer — and
34+
// doing so would clobber the enforcer that was already set up by a
35+
// concurrently-running team-mode server (relevant in tests).
36+
if !cfg.IsLocalMode() {
37+
if err := rbac.InitEnforcer(db, logger); err != nil {
38+
logger.Error("Failed to initialize RBAC", "error", err)
39+
panic(err)
40+
}
3441
}
3542
rbacProvider := rbac.NewDefaultProvider()
3643

‎internal/config/config.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,30 @@ func Load() (*Config, error) {
128128
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
129129
v.AutomaticEnv()
130130

131+
// viper's AutomaticEnv + Unmarshal does not propagate env vars into
132+
// nested structs without explicit BindEnv. Bind each nested key so that
133+
// e.g. NEBI_PACKAGE_MANAGER_PIXI_PATH overrides the pixi_path field.
134+
_ = v.BindEnv("package_manager.default_type", "NEBI_PACKAGE_MANAGER_DEFAULT_TYPE")
135+
_ = v.BindEnv("package_manager.pixi_path", "NEBI_PACKAGE_MANAGER_PIXI_PATH")
136+
_ = v.BindEnv("package_manager.uv_path", "NEBI_PACKAGE_MANAGER_UV_PATH")
137+
_ = v.BindEnv("storage.workspaces_dir", "NEBI_STORAGE_WORKSPACES_DIR")
138+
_ = v.BindEnv("server.host", "NEBI_SERVER_HOST")
139+
_ = v.BindEnv("server.port", "NEBI_SERVER_PORT")
140+
_ = v.BindEnv("server.mode", "NEBI_SERVER_MODE")
141+
_ = v.BindEnv("server.base_path", "NEBI_SERVER_BASE_PATH")
142+
_ = v.BindEnv("database.driver", "NEBI_DATABASE_DRIVER")
143+
_ = v.BindEnv("database.dsn", "NEBI_DATABASE_DSN")
144+
_ = v.BindEnv("auth.type", "NEBI_AUTH_TYPE")
145+
_ = v.BindEnv("auth.jwt_secret", "NEBI_AUTH_JWT_SECRET")
146+
_ = v.BindEnv("auth.oidc_issuer_url", "NEBI_AUTH_OIDC_ISSUER_URL")
147+
_ = v.BindEnv("auth.oidc_client_id", "NEBI_AUTH_OIDC_CLIENT_ID")
148+
_ = v.BindEnv("auth.oidc_client_secret", "NEBI_AUTH_OIDC_CLIENT_SECRET")
149+
_ = v.BindEnv("auth.oidc_redirect_url", "NEBI_AUTH_OIDC_REDIRECT_URL")
150+
_ = v.BindEnv("queue.type", "NEBI_QUEUE_TYPE")
151+
_ = v.BindEnv("queue.valkey_addr", "NEBI_QUEUE_VALKEY_ADDR")
152+
_ = v.BindEnv("log.format", "NEBI_LOG_FORMAT")
153+
_ = v.BindEnv("log.level", "NEBI_LOG_LEVEL")
154+
131155
var cfg Config
132156
if err := v.Unmarshal(&cfg); err != nil {
133157
return nil, fmt.Errorf("error unmarshaling config: %w", err)

0 commit comments

Comments
 (0)