feat(sourcemaps): Implement new tables supporting debug ids#44572
feat(sourcemaps): Implement new tables supporting debug ids#44572iambriccardo merged 17 commits intomasterfrom
Conversation
src/sentry/models/artifactbundle.py
Outdated
| class ArtifactBundle(Model): | ||
| __include_in_export__ = False | ||
|
|
||
| bundle_id = models.CharField(max_length=32, null=True) |
There was a problem hiding this comment.
| bundle_id = models.CharField(max_length=32, null=True) | |
| bundle_id = models.UUIDField(null=True) |
src/sentry/models/artifactbundle.py
Outdated
| bundle_id = models.CharField(max_length=32, null=True) | ||
| organization_id = BoundedBigIntegerField(db_index=True) | ||
| # TODO: define max length. | ||
| release_name = models.CharField(null=True) |
There was a problem hiding this comment.
Couldn't this also be an integer field, like dist_id?
If we're going with strings, use the same maxlength as on Release:
sentry/src/sentry/models/release.py
Line 478 in 7c58e7c
There was a problem hiding this comment.
Thanks! I was searching for the name of the release in the model and didn't think it could have been named version 😅.
There was a problem hiding this comment.
We cannot guarantee that a release exists yet. I think since we likely have one artifact per release, a string is fine. Alternatively one could deduplicate through another table but that seems pointless.
There was a problem hiding this comment.
I will add @beezz to check if the tables are fine.
src/sentry/models/artifactbundle.py
Outdated
|
|
||
|
|
||
| @region_silo_only_model | ||
| class ProjectArtifactBundleIndex(Model): |
There was a problem hiding this comment.
nit: I would drop the Index part of these intermediate tables, see for example
sentry/src/sentry/models/projectteam.py
Lines 43 to 47 in 7c58e7c
There was a problem hiding this comment.
I have followed the FileBlobIndex pattern but I agree, it's not an index. It's its own table.
|
@mitsuhiko we were discussing with @beezz about trying to use a hash index on the |
beezz
left a comment
There was a problem hiding this comment.
Please add the tables to the database router before merging so they are created on the files cluster since we have FK to the sentry.File table.
src/sentry/models/artifactbundle.py
Outdated
| ("organization_id", "bundle_id"), | ||
| ("organization_id", "release_name"), | ||
| ) | ||
| index_together = (("organization_id", "release_name"),) |
There was a problem hiding this comment.
You can remove this index since this set of columns is already in unique_together, which creates an index implicitly to enforce the uniqueness.
src/sentry/models/artifactbundle.py
Outdated
| class DebugIdArtifactBundle(Model): | ||
| __include_in_export__ = False | ||
|
|
||
| debug_id = models.UUIDField(unique=True, db_index=True) |
There was a problem hiding this comment.
Is there a requirement for having this as a separate uuid field? Is the implicit primary key id field not enough?
|
This PR has a migration; here is the generated SQL for --
-- Create model ArtifactBundle
--
CREATE TABLE "sentry_artifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "bundle_id" uuid NULL, "artifact_count" integer NOT NULL CHECK ("artifact_count" >= 0), "file_id" bigint NOT NULL);
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id" FOREIGN KEY ("file_id") REFERENCES "sentry_file" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_artifactbundle" VALIDATE CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" ON "sentry_artifactbundle" ("organization_id", "bundle_id");
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" UNIQUE USING INDEX "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq";
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_b5a13770" ON "sentry_artifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_file_id_1d6752c9" ON "sentry_artifactbundle" ("file_id");
--
-- Create model ReleaseArtifactBundle
--
CREATE TABLE "sentry_releaseartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "release_name" varchar(250) NOT NULL, "dist_id" bigint NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_releaseartifactbundle" VALIDATE CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" ON "sentry_releaseartifactbundle" ("organization_id", "release_name", "dist_id", "artifact_bundle_id");
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" UNIQUE USING INDEX "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq";
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_organization_id_10fc7929" ON "sentry_releaseartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_artifact_bundle_id_e0298a12" ON "sentry_releaseartifactbundle" ("artifact_bundle_id");
--
-- Create model ProjectArtifactBundle
--
CREATE TABLE "sentry_projectartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "project_id" bigint NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_projectartifactbundle" VALIDATE CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" ON "sentry_projectartifactbundle" ("project_id", "artifact_bundle_id");
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" UNIQUE USING INDEX "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq";
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_project_id_7588084b" ON "sentry_projectartifactbundle" ("project_id");
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_artifact_bundle_id_c3f68015" ON "sentry_projectartifactbundle" ("artifact_bundle_id");
--
-- Create model DebugIdArtifactBundle
--
CREATE TABLE "sentry_debugidartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "debug_id" uuid NOT NULL, "source_file_type" integer NOT NULL, "date_added" timestamp with time zone NOT NULL, "date_last_accessed" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_debugidartifactbundle" VALIDATE CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" ON "sentry_debugidartifactbundle" ("debug_id", "artifact_bundle_id", "source_file_type");
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" UNIQUE USING INDEX "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq";
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_organization_id_4db980bf" ON "sentry_debugidartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_artifact_bundle_id_91f3140f" ON "sentry_debugidartifactbundle" ("artifact_bundle_id"); |
Related to this - the files DB cluster does not automatically run migrations and you'll need to have those migrations deployed manually, or temporarily update the |
|
@markstory I had already a PR open for that, it is in draft because we first wanted to clarify the required tables. It is available here: https://github.com/getsentry/getsentry/pull/9593 @beezz and I will sync on how we want to roll this out properly. |
| from .apiscopes import * # NOQA | ||
| from .apitoken import * # NOQA | ||
| from .appconnectbuilds import * # NOQA | ||
| from .artifactbundle import * # NOQA |
There was a problem hiding this comment.
@asottile-sentry do we still need this? I really want to at least avoid new models to end up in here.
There was a problem hiding this comment.
I don't think so -- I think this part of the patch can be discarded
| __include_in_export__ = False | ||
|
|
||
| organization_id = BoundedBigIntegerField(db_index=True) | ||
| release_name = models.CharField(max_length=250) |
There was a problem hiding this comment.
Should this be release_id and related to the Release table?
There was a problem hiding this comment.
@mitsuhiko wanted a weak reference to releases and decided to use the release.version.
|
This PR has a migration; here is the generated SQL for --
-- Create model ArtifactBundle
--
CREATE TABLE "sentry_artifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "bundle_id" uuid NULL, "artifact_count" integer NOT NULL CHECK ("artifact_count" >= 0), "date_added" timestamp with time zone NOT NULL, "file_id" bigint NOT NULL);
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id" FOREIGN KEY ("file_id") REFERENCES "sentry_file" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_artifactbundle" VALIDATE CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" ON "sentry_artifactbundle" ("organization_id", "bundle_id");
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" UNIQUE USING INDEX "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq";
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_b5a13770" ON "sentry_artifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_file_id_1d6752c9" ON "sentry_artifactbundle" ("file_id");
--
-- Create model ReleaseArtifactBundle
--
CREATE TABLE "sentry_releaseartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "release_name" varchar(250) NOT NULL, "dist_id" bigint NULL, "date_added" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_releaseartifactbundle" VALIDATE CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" ON "sentry_releaseartifactbundle" ("organization_id", "release_name", "dist_id", "artifact_bundle_id");
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" UNIQUE USING INDEX "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq";
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_organization_id_10fc7929" ON "sentry_releaseartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_artifact_bundle_id_e0298a12" ON "sentry_releaseartifactbundle" ("artifact_bundle_id");
--
-- Create model ProjectArtifactBundle
--
CREATE TABLE "sentry_projectartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "project_id" bigint NOT NULL, "date_added" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_projectartifactbundle" VALIDATE CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" ON "sentry_projectartifactbundle" ("project_id", "artifact_bundle_id");
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" UNIQUE USING INDEX "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq";
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_organization_id_a4b9d5f9" ON "sentry_projectartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_project_id_7588084b" ON "sentry_projectartifactbundle" ("project_id");
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_artifact_bundle_id_c3f68015" ON "sentry_projectartifactbundle" ("artifact_bundle_id");
--
-- Create model DebugIdArtifactBundle
--
CREATE TABLE "sentry_debugidartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "debug_id" uuid NOT NULL, "source_file_type" integer NOT NULL, "date_added" timestamp with time zone NOT NULL, "date_last_accessed" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_debugidartifactbundle" VALIDATE CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" ON "sentry_debugidartifactbundle" ("debug_id", "artifact_bundle_id", "source_file_type");
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" UNIQUE USING INDEX "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq";
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_organization_id_4db980bf" ON "sentry_debugidartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_artifact_bundle_id_91f3140f" ON "sentry_debugidartifactbundle" ("artifact_bundle_id"); |
* master: (79 commits) feat(perf-issues): Add performance issue detection timing runner command (#44912) Revert "chore: Investigating org slug already set to a different value (#45134)" fix(hybrid-cloud): Redirect to org restoration page for customer domains (#45159) bug(replays): Fix 500 error when marshaling tags field (#45097) ref(sourcemaps): Redesign lookup of source and sourcemaps (#45032) chore: Investigating org slug already set to a different value (#45134) feat(dynamic-sampling): Implement prioritize by project bias [TET-574] (#42939) feat(dynamic-sampling): Add transaction name prioritize option - (#45034) feat(dyn-sampling): add new bias toggle to project details for prioritise by tx name [TET-717] (#44944) feat(admin) Add admin relay project config view [TET-509] (#45120) Revert "chore(assignment): Add analytics when autoassigning after a manual assignment (#45099)" feat(sourcemaps): Implement new tables supporting debug ids (#44572) ref(js): Remove usage of react-document-title (#45170) chore(py): Consistently name urls using `organization-` prefix (#45180) ref: rename acceptance required checks collector (#45156) chore(assignment): Add analytics when autoassigning after a manual assignment (#45099) feat(source-maps): Update copy for source map debug alerts (#45164) ref(js): Remove custom usage of DocumentTitle (#45165) chore(login): update the login banners (#45151) ref(py): Remove one more legacy project_id from Environment (#45160) ...
This PR aims at implementing the new tables used to support debug ids for source maps.