From f92577b76ab91582bcd1207423a5138914168096 Mon Sep 17 00:00:00 2001 From: Andrew Lalis Date: Thu, 16 Feb 2023 18:40:19 +0100 Subject: [PATCH] Added more data to user index. --- gymboard-search/README.md | 2 +- .../gymboardsearch/GymboardSearchApplication.java | 11 +++++++---- .../gymboardsearch/dto/UserResponse.java | 10 ++++++++-- .../gymboardsearch/index/IndexComponents.java | 15 ++++++++++++++- .../gymboardsearch/index/JdbcIndexGenerator.java | 10 +++++++--- .../resources/application-development.properties | 4 ++++ .../src/main/resources/sql/select-users.sql | 9 ++++++++- 7 files changed, 49 insertions(+), 12 deletions(-) diff --git a/gymboard-search/README.md b/gymboard-search/README.md index 7f7f5e7..082e3dc 100644 --- a/gymboard-search/README.md +++ b/gymboard-search/README.md @@ -6,4 +6,4 @@ This application is configured with read-only access to the central Gymboard dat ## Developing -Currently, this application is designed to boot up and immediately read the latest data from the Gymboard API's database to rebuild its indexes. +Currently, this application is designed to boot up and immediately read the latest data from the Gymboard API's database to rebuild its indexes, then continue to do so at scheduled intervals. diff --git a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/GymboardSearchApplication.java b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/GymboardSearchApplication.java index ece26c4..00502de 100644 --- a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/GymboardSearchApplication.java +++ b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/GymboardSearchApplication.java @@ -1,14 +1,17 @@ package nl.andrewlalis.gymboardsearch; import nl.andrewlalis.gymboardsearch.index.JdbcIndexGenerator; -import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; import java.util.TimeZone; +import java.util.concurrent.TimeUnit; @SpringBootApplication -public class GymboardSearchApplication implements CommandLineRunner { +@EnableScheduling +public class GymboardSearchApplication { public GymboardSearchApplication(JdbcIndexGenerator gymIndexGenerator, JdbcIndexGenerator userIndexGenerator) { this.gymIndexGenerator = gymIndexGenerator; @@ -23,8 +26,8 @@ public class GymboardSearchApplication implements CommandLineRunner { private final JdbcIndexGenerator gymIndexGenerator; private final JdbcIndexGenerator userIndexGenerator; - @Override - public void run(String... args) { + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.HOURS) + public void reIndex() { gymIndexGenerator.generate(); userIndexGenerator.generate(); } diff --git a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/dto/UserResponse.java b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/dto/UserResponse.java index dc1cae7..f5eb05c 100644 --- a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/dto/UserResponse.java +++ b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/dto/UserResponse.java @@ -4,12 +4,18 @@ import org.apache.lucene.document.Document; public record UserResponse( String id, - String name + String name, + long submissionCount, + boolean accountPrivate, + String locale ) { public UserResponse(Document doc) { this( doc.get("id"), - doc.get("name") + doc.get("name"), + doc.getField("submission_count").numericValue().longValue(), + Boolean.parseBoolean(doc.get("account_private")), + doc.get("locale") ); } } diff --git a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/IndexComponents.java b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/IndexComponents.java index e8b3203..b086f7c 100644 --- a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/IndexComponents.java +++ b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/IndexComponents.java @@ -3,6 +3,7 @@ package nl.andrewlalis.gymboardsearch.index; import nl.andrewlalis.gymboardsearch.dto.GymResponse; import nl.andrewlalis.gymboardsearch.dto.UserResponse; import org.apache.lucene.document.*; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; @@ -18,9 +19,18 @@ import java.sql.DriverManager; */ @Component public class IndexComponents { + @Value("${app.db.jdbc-url}") + private String jdbcUrl; + + @Value("${app.db.user}") + private String dbUser; + + @Value("${app.db.password}") + private String dbPassword; + @Bean public JdbcConnectionSupplier jdbcConnectionSupplier() { - return () -> DriverManager.getConnection("jdbc:postgresql://localhost:5432/gymboard-api-dev", "gymboard-api-dev", "testpass"); + return () -> DriverManager.getConnection(jdbcUrl, dbUser, dbPassword); } @Bean @@ -33,6 +43,9 @@ public class IndexComponents { var doc = new Document(); doc.add(new StoredField("id", rs.getString("id"))); doc.add(new TextField("name", rs.getString("name"), Field.Store.YES)); + doc.add(new StoredField("submission_count", rs.getLong("submission_count"))); + doc.add(new StringField("locale", rs.getString("locale"), Field.Store.YES)); + doc.add(new StringField("account_private", String.valueOf(rs.getBoolean("account_private")), Field.Store.YES)); return doc; } ); diff --git a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/JdbcIndexGenerator.java b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/JdbcIndexGenerator.java index 48e3610..657da44 100644 --- a/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/JdbcIndexGenerator.java +++ b/gymboard-search/src/main/java/nl/andrewlalis/gymboardsearch/index/JdbcIndexGenerator.java @@ -36,7 +36,7 @@ public class JdbcIndexGenerator { if (Files.exists(indexDir)) { try { FileSystemUtils.deleteRecursively(indexDir); - Files.createDirectory(indexDir); + Files.createDirectories(indexDir); } catch (IOException e) { log.error("Failed to reset index directory.", e); return; @@ -47,9 +47,12 @@ public class JdbcIndexGenerator { ResultSet rs = resultSetSupplier.supply(conn); Analyzer analyzer = new StandardAnalyzer(); - Directory luceneDir = FSDirectory.open(indexDir); - IndexWriter indexWriter = new IndexWriter(luceneDir, new IndexWriterConfig(analyzer)) + Directory luceneDir = FSDirectory.open(indexDir) ) { + IndexWriterConfig config = new IndexWriterConfig(analyzer); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); + IndexWriter indexWriter = new IndexWriter(luceneDir, config); + long count = 0; while (rs.next()) { try { @@ -60,6 +63,7 @@ public class JdbcIndexGenerator { } } log.info("Indexed {} entities.", count); + indexWriter.close(); } catch (Exception e) { log.error("Failed to prepare indexing components.", e); } diff --git a/gymboard-search/src/main/resources/application-development.properties b/gymboard-search/src/main/resources/application-development.properties index 2a200e5..369ac3f 100644 --- a/gymboard-search/src/main/resources/application-development.properties +++ b/gymboard-search/src/main/resources/application-development.properties @@ -1,2 +1,6 @@ server.port=8081 app.web-origin=http://localhost:9000 + +app.db.jdbc-url=jdbc:postgresql://localhost:5432/gymboard-api-dev +app.db.user=gymboard-api-dev +app.db.password=testpass diff --git a/gymboard-search/src/main/resources/sql/select-users.sql b/gymboard-search/src/main/resources/sql/select-users.sql index 3298bcf..6df9427 100644 --- a/gymboard-search/src/main/resources/sql/select-users.sql +++ b/gymboard-search/src/main/resources/sql/select-users.sql @@ -1,7 +1,14 @@ SELECT u.id as id, u.email as email, - u.name as name + u.name as name, + ( + SELECT COUNT(id) + FROM submission + WHERE submission.id = u.id + ) as submission_count, + p.account_private as account_private, + p.locale as locale FROM auth_user u LEFT JOIN auth_user_preferences p ON u.id = p.user_id WHERE u.activated = TRUE