[breaking] improved DB lookups.

RillingDev · May 23, 2023 · 56cfb72 · 56cfb72
1 parent e02b78c
commit 56cfb72
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 60 deletions.
diff --git a/src/main/java/dev/rilling/musicbrainzenricher/MusicbrainzEnricherService.java b/src/main/java/dev/rilling/musicbrainzenricher/MusicbrainzEnricherService.java
@@ -31,14 +31,13 @@ public class MusicbrainzEnricherService {
 
 	public void runInAutoQueryMode(@NotNull DataType dataType) {
 		switch (dataType) {
-			case RELEASE -> musicbrainzAutoQueryService.autoQueryReleasesWithRelationships(mbid -> executeEnrichment(
+			case RELEASE -> musicbrainzAutoQueryService.autoQueryReleases(mbid -> executeEnrichment(
 				dataType,
 				mbid,
 				findFittingEnrichmentService(dataType)));
-			case RELEASE_GROUP ->
-				musicbrainzAutoQueryService.autoQueryReleaseGroupsWithRelationships(mbid -> executeEnrichment(dataType,
-					mbid,
-					findFittingEnrichmentService(dataType)));
+			case RELEASE_GROUP -> musicbrainzAutoQueryService.autoQueryReleaseGroups(mbid -> executeEnrichment(dataType,
+				mbid,
+				findFittingEnrichmentService(dataType)));
 		}
 	}
 

diff --git a/...ain/java/dev/rilling/musicbrainzenricher/api/musicbrainz/MusicbrainzAutoQueryService.java b/...ain/java/dev/rilling/musicbrainzenricher/api/musicbrainz/MusicbrainzAutoQueryService.java
@@ -28,27 +28,21 @@ public class MusicbrainzAutoQueryService {
 		this.releaseGroupRepository = releaseGroupRepository;
 	}
 
-	public void autoQueryReleasesWithRelationships(@NotNull Consumer<UUID> mbidConsumer) {
-		long count = releaseRepository.countNewReleasesWhereRelationshipsExist();
-		LOGGER.info("Found a total of {} new auto query releases.", count);
-
-		long offset = 0;
-		while (offset < count) {
-			LOGGER.info("Loading {} releases with offset {}...", LIMIT, offset);
-			releaseRepository.findNewReleaseMbidWhereRelationshipsExist(offset, LIMIT).forEach(mbidConsumer);
-			offset += LIMIT;
+	public void autoQueryReleases(@NotNull Consumer<UUID> mbidConsumer) {
+		long count = releaseRepository.countFromWorkQueue();
+		while (count > 0) {
+			LOGGER.info("{} auto query releases remaining.", count);
+			releaseRepository.findFromWorkQueue(LIMIT).forEach(mbidConsumer);
+			count = releaseRepository.countFromWorkQueue();
 		}
 	}
 
-	public void autoQueryReleaseGroupsWithRelationships(@NotNull Consumer<UUID> mbidConsumer) {
-		long count = releaseGroupRepository.countNewReleaseGroupsWhereRelationshipsExist();
-		LOGGER.info("Found a total of {} new auto query release groups.", count);
-
-		long offset = 0;
-		while (offset < count) {
-			LOGGER.info("Loading {} release groups with offset {}...", LIMIT, offset);
-			releaseGroupRepository.findNewReleaseGroupsMbidWhereRelationshipsExist(offset, LIMIT).forEach(mbidConsumer);
-			offset += LIMIT;
+	public void autoQueryReleaseGroups(@NotNull Consumer<UUID> mbidConsumer) {
+		long count = releaseGroupRepository.countFromWorkQueue();
+		while (count > 0) {
+			LOGGER.info("{} auto query release groups remaining.", count);
+			releaseGroupRepository.findFromWorkQueue(LIMIT).forEach(mbidConsumer);
+			count = releaseGroupRepository.countFromWorkQueue();
 		}
 	}
 

diff --git a/src/main/java/dev/rilling/musicbrainzenricher/core/ReleaseGroupRepository.java b/src/main/java/dev/rilling/musicbrainzenricher/core/ReleaseGroupRepository.java
@@ -20,28 +20,13 @@ public class ReleaseGroupRepository {
 		this.jdbcTemplate = jdbcTemplate;
 	}
 
-	public long countNewReleaseGroupsWhereRelationshipsExist() {
-		return Objects.requireNonNull(jdbcTemplate.queryForObject("""
-			SELECT COUNT(*)	FROM musicbrainz.release_group rg
-				WHERE rg.id IN
-					(SELECT lrgu.entity0 FROM musicbrainz.l_release_group_url lrgu)
-				AND rg.gid NOT IN
-				  (SELECT rghe.release_group_gid FROM musicbrainz_enricher.release_group_history_entry rghe)
-			""", Long.class));
+	public long countFromWorkQueue() {
+		return Objects.requireNonNull(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM musicbrainz_enricher.release_group_work_queue", Long.class));
 	}
 
 	@NotNull
-	public List<UUID> findNewReleaseGroupsMbidWhereRelationshipsExist(long offset, int limit) {
-		List<UUID> mbids = jdbcTemplate.query("""
-			SELECT rg.gid
-			FROM musicbrainz.release_group rg
-				WHERE rg.id IN
-					  (SELECT lrgu.entity0 FROM musicbrainz.l_release_group_url	lrgu)
-				AND rg.gid NOT IN
-					(SELECT rghe.release_group_gid FROM musicbrainz_enricher.release_group_history_entry rghe)
-			ORDER BY rg.id
-			OFFSET ? LIMIT ?
-			""", (rs, rowNum) -> rs.getObject("gid", UUID.class), offset, limit);
+	public List<UUID> findFromWorkQueue(long offset) {
+		List<UUID> mbids = jdbcTemplate.query("SELECT rg.gid FROM musicbrainz_enricher.release_group_work_queue rg LIMIT ?", (rs, rowNum) -> rs.getObject("gid", UUID.class), offset);
 		return Collections.unmodifiableList(mbids);
 	}
 }
diff --git a/src/main/java/dev/rilling/musicbrainzenricher/core/ReleaseRepository.java b/src/main/java/dev/rilling/musicbrainzenricher/core/ReleaseRepository.java
@@ -19,27 +19,13 @@ public class ReleaseRepository {
 		this.jdbcTemplate = jdbcTemplate;
 	}
 
-	public long countNewReleasesWhereRelationshipsExist() {
-		return Objects.requireNonNull(jdbcTemplate.queryForObject("""
-			SELECT COUNT(*) FROM musicbrainz.release r
-				WHERE r.id IN
-					(SELECT lru.entity0 FROM musicbrainz.l_release_url lru)
-				AND r.gid NOT IN
-				  (SELECT rhe.release_gid FROM musicbrainz_enricher.release_history_entry rhe)
-			""", Long.class));
+	public long countFromWorkQueue() {
+		return Objects.requireNonNull(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM musicbrainz_enricher.release_work_queue", Long.class));
 	}
 
 	@NotNull
-	public List<UUID> findNewReleaseMbidWhereRelationshipsExist(long offset, int limit) {
-		List<UUID> mbids = jdbcTemplate.query("""
-			SELECT r.gid FROM musicbrainz.release r
-				WHERE r.id IN
-					(SELECT lru.entity0 FROM musicbrainz.l_release_url lru)
-				AND r.gid NOT IN
-				  (SELECT rhe.release_gid FROM musicbrainz_enricher.release_history_entry rhe)
-			ORDER BY r.id
-			OFFSET ? LIMIT ?
-			""", (rs, rowNum) -> rs.getObject("gid", UUID.class), offset, limit);
+	public List<UUID> findFromWorkQueue(int limit) {
+		List<UUID> mbids = jdbcTemplate.query("SELECT r.gid FROM musicbrainz_enricher.release_work_queue r LIMIT ?", (rs, rowNum) -> rs.getObject("gid", UUID.class), limit);
 		return Collections.unmodifiableList(mbids);
 	}
 }
diff --git a/src/main/resources/db/migration/V1_0__create_history_schema.sql b/src/main/resources/db/migration/V1_0__create_history_schema.sql
@@ -4,8 +4,31 @@ CREATE TABLE musicbrainz_enricher.release_history_entry
 		ON DELETE CASCADE
 );
 
+CREATE MATERIALIZED VIEW musicbrainz_enricher.release_with_relationships AS
+SELECT *
+FROM musicbrainz.release r
+WHERE r.id IN
+	  (SELECT lru.entity0 FROM musicbrainz.l_release_url lru);
+
+CREATE VIEW musicbrainz_enricher.release_work_queue AS
+SELECT *
+FROM musicbrainz_enricher.release_with_relationships r
+WHERE r.gid NOT IN (SELECT rhe.release_gid FROM musicbrainz_enricher.release_history_entry rhe);
+
+
 CREATE TABLE musicbrainz_enricher.release_group_history_entry
 (
 	release_group_gid UUID PRIMARY KEY NOT NULL REFERENCES musicbrainz.release_group (gid)
 		ON DELETE CASCADE
 );
+
+CREATE MATERIALIZED VIEW musicbrainz_enricher.release_group_with_relationships AS
+SELECT *
+FROM musicbrainz.release_group rg
+WHERE rg.id IN
+	  (SELECT lrgu.entity0 FROM musicbrainz.l_release_group_url lrgu);
+
+CREATE VIEW musicbrainz_enricher.release_group_work_queue AS
+SELECT *
+FROM musicbrainz_enricher.release_group_with_relationships rg
+WHERE rg.gid NOT IN (SELECT rghe.release_group_gid FROM musicbrainz_enricher.release_group_history_entry rghe);