Refactored indexing and searching to be modular.
This commit is contained in:
parent
a8715fa8d2
commit
ee5ff41167
|
@ -28,6 +28,11 @@ public class UserController {
|
||||||
return new UserResponse(user);
|
return new UserResponse(user);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping(path = "/auth/users/{userId}")
|
||||||
|
public UserResponse getUser(@PathVariable String userId) {
|
||||||
|
return userService.getUser(userId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Endpoint for updating one's own password.
|
* Endpoint for updating one's own password.
|
||||||
* @param user The user that's updating their password.
|
* @param user The user that's updating their password.
|
||||||
|
|
|
@ -7,18 +7,18 @@ import java.time.format.DateTimeFormatter;
|
||||||
public record UserPersonalDetailsResponse(
|
public record UserPersonalDetailsResponse(
|
||||||
String userId,
|
String userId,
|
||||||
String birthDate,
|
String birthDate,
|
||||||
float currentWeight,
|
Float currentWeight,
|
||||||
String currentWeightUnit,
|
String currentWeightUnit,
|
||||||
float currentMetricWeight,
|
Float currentMetricWeight,
|
||||||
String sex
|
String sex
|
||||||
) {
|
) {
|
||||||
public UserPersonalDetailsResponse(UserPersonalDetails pd) {
|
public UserPersonalDetailsResponse(UserPersonalDetails pd) {
|
||||||
this(
|
this(
|
||||||
pd.getUserId(),
|
pd.getUserId(),
|
||||||
pd.getBirthDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
|
pd.getBirthDate() == null ? null : pd.getBirthDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
|
||||||
pd.getCurrentWeight().floatValue(),
|
pd.getCurrentWeight() == null ? null : pd.getCurrentWeight().floatValue(),
|
||||||
pd.getCurrentWeightUnit().name(),
|
pd.getCurrentWeightUnit() == null ? null : pd.getCurrentWeightUnit().name(),
|
||||||
pd.getCurrentMetricWeight().floatValue(),
|
pd.getCurrentMetricWeight() == null ? null : pd.getCurrentMetricWeight().floatValue(),
|
||||||
pd.getSex().name()
|
pd.getSex().name()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
import { api } from 'src/api/main/index';
|
import {api} from 'src/api/main/index';
|
||||||
import { AuthStoreType } from 'stores/auth-store';
|
import {AuthStoreType} from 'stores/auth-store';
|
||||||
import Timeout = NodeJS.Timeout;
|
import Timeout = NodeJS.Timeout;
|
||||||
|
|
||||||
export interface User {
|
export interface User {
|
||||||
|
@ -9,6 +9,26 @@ export interface User {
|
||||||
name: string;
|
name: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export enum PersonSex {
|
||||||
|
MALE = 'MALE',
|
||||||
|
FEMALE = 'FEMALE',
|
||||||
|
UNKNOWN = 'UNKNOWN'
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UserPersonalDetails {
|
||||||
|
userId: string;
|
||||||
|
birthDate?: string;
|
||||||
|
currentWeight?: number;
|
||||||
|
currentWeightUnit?: number;
|
||||||
|
sex: PersonSex;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UserPreferences {
|
||||||
|
userId: string;
|
||||||
|
accountPrivate: boolean;
|
||||||
|
locale: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface TokenCredentials {
|
export interface TokenCredentials {
|
||||||
email: string;
|
email: string;
|
||||||
password: string;
|
password: string;
|
||||||
|
@ -66,6 +86,11 @@ class AuthModule {
|
||||||
return response.data;
|
return response.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async fetchUser(userId: string, authStore: AuthStoreType): Promise<User> {
|
||||||
|
const response = await api.get(`/auth/users/${userId}`, authStore.axiosConfig);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
||||||
public async updatePassword(newPassword: string, authStore: AuthStoreType) {
|
public async updatePassword(newPassword: string, authStore: AuthStoreType) {
|
||||||
await api.post(
|
await api.post(
|
||||||
'/auth/me/password',
|
'/auth/me/password',
|
||||||
|
@ -84,6 +109,16 @@ class AuthModule {
|
||||||
newPassword: newPassword,
|
newPassword: newPassword,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async getMyPersonalDetails(authStore: AuthStoreType): Promise<UserPersonalDetails> {
|
||||||
|
const response = await api.get('/auth/me/personal-details', authStore.axiosConfig);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getMyPreferences(authStore: AuthStoreType): Promise<UserPreferences> {
|
||||||
|
const response = await api.get('/auth/me/preferences', authStore.axiosConfig);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default AuthModule;
|
export default AuthModule;
|
||||||
|
|
|
@ -33,3 +33,4 @@ build/
|
||||||
.vscode/
|
.vscode/
|
||||||
|
|
||||||
gym-index/
|
gym-index/
|
||||||
|
user-index/
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package nl.andrewlalis.gymboardsearch;
|
package nl.andrewlalis.gymboardsearch;
|
||||||
|
|
||||||
import nl.andrewlalis.gymboardsearch.index.GymIndexGenerator;
|
import nl.andrewlalis.gymboardsearch.index.JdbcIndexGenerator;
|
||||||
import org.springframework.boot.CommandLineRunner;
|
import org.springframework.boot.CommandLineRunner;
|
||||||
import org.springframework.boot.SpringApplication;
|
import org.springframework.boot.SpringApplication;
|
||||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
@ -9,10 +9,10 @@ import java.util.TimeZone;
|
||||||
|
|
||||||
@SpringBootApplication
|
@SpringBootApplication
|
||||||
public class GymboardSearchApplication implements CommandLineRunner {
|
public class GymboardSearchApplication implements CommandLineRunner {
|
||||||
private final GymIndexGenerator gymIndexGenerator;
|
|
||||||
|
|
||||||
public GymboardSearchApplication(GymIndexGenerator gymIndexGenerator) {
|
public GymboardSearchApplication(JdbcIndexGenerator gymIndexGenerator, JdbcIndexGenerator userIndexGenerator) {
|
||||||
this.gymIndexGenerator = gymIndexGenerator;
|
this.gymIndexGenerator = gymIndexGenerator;
|
||||||
|
this.userIndexGenerator = userIndexGenerator;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
|
@ -20,8 +20,12 @@ public class GymboardSearchApplication implements CommandLineRunner {
|
||||||
SpringApplication.run(GymboardSearchApplication.class, args);
|
SpringApplication.run(GymboardSearchApplication.class, args);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private final JdbcIndexGenerator gymIndexGenerator;
|
||||||
|
private final JdbcIndexGenerator userIndexGenerator;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run(String... args) throws Exception {
|
public void run(String... args) {
|
||||||
gymIndexGenerator.generateIndex();
|
gymIndexGenerator.generate();
|
||||||
|
userIndexGenerator.generate();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
package nl.andrewlalis.gymboardsearch;
|
package nl.andrewlalis.gymboardsearch;
|
||||||
|
|
||||||
import nl.andrewlalis.gymboardsearch.dto.GymResponse;
|
import nl.andrewlalis.gymboardsearch.dto.GymResponse;
|
||||||
import nl.andrewlalis.gymboardsearch.index.GymIndexSearcher;
|
import nl.andrewlalis.gymboardsearch.dto.UserResponse;
|
||||||
|
import nl.andrewlalis.gymboardsearch.index.QueryIndexSearcher;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
@ -10,14 +11,21 @@ import java.util.List;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
public class SearchController {
|
public class SearchController {
|
||||||
private final GymIndexSearcher gymIndexSearcher;
|
private final QueryIndexSearcher<GymResponse> gymIndexSearcher;
|
||||||
|
private final QueryIndexSearcher<UserResponse> userIndexSearcher;
|
||||||
|
|
||||||
public SearchController(GymIndexSearcher gymIndexSearcher) {
|
public SearchController(QueryIndexSearcher<GymResponse> gymIndexSearcher, QueryIndexSearcher<UserResponse> userIndexSearcher) {
|
||||||
this.gymIndexSearcher = gymIndexSearcher;
|
this.gymIndexSearcher = gymIndexSearcher;
|
||||||
|
this.userIndexSearcher = userIndexSearcher;
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping(path = "/search/gyms")
|
@GetMapping(path = "/search/gyms")
|
||||||
public List<GymResponse> searchGyms(@RequestParam(name = "q", required = false) String query) {
|
public List<GymResponse> searchGyms(@RequestParam(name = "q", required = false) String query) {
|
||||||
return gymIndexSearcher.searchGyms(query);
|
return gymIndexSearcher.search(query);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping(path = "/search/users")
|
||||||
|
public List<UserResponse> searchUsers(@RequestParam(name = "q", required = false) String query) {
|
||||||
|
return userIndexSearcher.search(query);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,15 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.dto;
|
||||||
|
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
|
||||||
|
public record UserResponse(
|
||||||
|
String id,
|
||||||
|
String name
|
||||||
|
) {
|
||||||
|
public UserResponse(Document doc) {
|
||||||
|
this(
|
||||||
|
doc.get("id"),
|
||||||
|
doc.get("name")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,74 +0,0 @@
|
||||||
package nl.andrewlalis.gymboardsearch.index;
|
|
||||||
|
|
||||||
import nl.andrewlalis.gymboardsearch.DbUtils;
|
|
||||||
import org.apache.lucene.analysis.Analyzer;
|
|
||||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
|
||||||
import org.apache.lucene.document.*;
|
|
||||||
import org.apache.lucene.index.IndexWriter;
|
|
||||||
import org.apache.lucene.index.IndexWriterConfig;
|
|
||||||
import org.apache.lucene.store.Directory;
|
|
||||||
import org.apache.lucene.store.FSDirectory;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.util.FileSystemUtils;
|
|
||||||
|
|
||||||
import java.math.BigDecimal;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.sql.Connection;
|
|
||||||
import java.sql.DriverManager;
|
|
||||||
import java.sql.PreparedStatement;
|
|
||||||
import java.sql.ResultSet;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class GymIndexGenerator {
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(GymIndexGenerator.class);
|
|
||||||
|
|
||||||
public void generateIndex() throws Exception {
|
|
||||||
log.info("Starting Gym index generation.");
|
|
||||||
Path gymIndexDir = Path.of("gym-index");
|
|
||||||
FileSystemUtils.deleteRecursively(gymIndexDir);
|
|
||||||
Files.createDirectory(gymIndexDir);
|
|
||||||
long count = 0;
|
|
||||||
try (
|
|
||||||
Connection conn = DriverManager.getConnection("jdbc:postgresql://localhost:5432/gymboard-api-dev", "gymboard-api-dev", "testpass");
|
|
||||||
PreparedStatement stmt = conn.prepareStatement(DbUtils.loadClasspathString("/sql/select-gyms.sql"));
|
|
||||||
ResultSet resultSet = stmt.executeQuery();
|
|
||||||
|
|
||||||
Analyzer analyzer = new StandardAnalyzer();
|
|
||||||
Directory indexDir = FSDirectory.open(gymIndexDir);
|
|
||||||
IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(analyzer))
|
|
||||||
) {
|
|
||||||
while (resultSet.next()) {
|
|
||||||
String shortName = resultSet.getString("short_name");
|
|
||||||
String displayName = resultSet.getString("display_name");
|
|
||||||
String cityShortName = resultSet.getString("city_short_name");
|
|
||||||
String cityName = resultSet.getString("city_name");
|
|
||||||
String countryCode = resultSet.getString("country_code");
|
|
||||||
String countryName = resultSet.getString("country_name");
|
|
||||||
String streetAddress = resultSet.getString("street_address");
|
|
||||||
BigDecimal latitude = resultSet.getBigDecimal("latitude");
|
|
||||||
BigDecimal longitude = resultSet.getBigDecimal("longitude");
|
|
||||||
String gymCompoundId = String.format("%s_%s_%s", countryCode, cityShortName, shortName);
|
|
||||||
|
|
||||||
Document doc = new Document();
|
|
||||||
doc.add(new StoredField("compound_id", gymCompoundId));
|
|
||||||
doc.add(new TextField("short_name", shortName, Field.Store.YES));
|
|
||||||
doc.add(new TextField("display_name", displayName, Field.Store.YES));
|
|
||||||
doc.add(new TextField("city_short_name", cityShortName, Field.Store.YES));
|
|
||||||
doc.add(new TextField("city_name", cityName, Field.Store.YES));
|
|
||||||
doc.add(new TextField("country_code", countryCode, Field.Store.YES));
|
|
||||||
doc.add(new TextField("country_name", countryName, Field.Store.YES));
|
|
||||||
doc.add(new TextField("street_address", streetAddress, Field.Store.YES));
|
|
||||||
doc.add(new DoublePoint("latitude_point", latitude.doubleValue()));
|
|
||||||
doc.add(new StoredField("latitude", latitude.doubleValue()));
|
|
||||||
doc.add(new DoublePoint("longitude_point", longitude.doubleValue()));
|
|
||||||
doc.add(new StoredField("longitude", longitude.doubleValue()));
|
|
||||||
indexWriter.addDocument(doc);
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.info("Gym index generation complete. {} gyms indexed.", count);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,63 +0,0 @@
|
||||||
package nl.andrewlalis.gymboardsearch.index;
|
|
||||||
|
|
||||||
import nl.andrewlalis.gymboardsearch.dto.GymResponse;
|
|
||||||
import org.apache.lucene.document.Document;
|
|
||||||
import org.apache.lucene.index.DirectoryReader;
|
|
||||||
import org.apache.lucene.index.Term;
|
|
||||||
import org.apache.lucene.search.*;
|
|
||||||
import org.apache.lucene.store.FSDirectory;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.util.*;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Searcher that uses a Lucene {@link IndexSearcher} to search for gyms using
|
|
||||||
* a query that's built from a weighted list of wildcard search terms.
|
|
||||||
* <ol>
|
|
||||||
* <li>If the query is blank, return an empty list.</li>
|
|
||||||
* <li>Split the query into words, append the wildcard '*' to each word.</li>
|
|
||||||
* <li>For each word, add a boosted wildcard query for each weighted field.</li>
|
|
||||||
* </ol>
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class GymIndexSearcher {
|
|
||||||
public List<GymResponse> searchGyms(String rawQuery) {
|
|
||||||
if (rawQuery == null || rawQuery.isBlank()) return Collections.emptyList();
|
|
||||||
String[] terms = rawQuery.split("\\s+");
|
|
||||||
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
|
|
||||||
Map<String, Float> fieldWeights = new HashMap<>();
|
|
||||||
fieldWeights.put("short_name", 3f);
|
|
||||||
fieldWeights.put("display_name", 3f);
|
|
||||||
fieldWeights.put("city_short_name", 1f);
|
|
||||||
fieldWeights.put("city_name", 1f);
|
|
||||||
fieldWeights.put("country_code", 0.25f);
|
|
||||||
fieldWeights.put("country_name", 0.5f);
|
|
||||||
fieldWeights.put("street_address", 0.1f);
|
|
||||||
for (String term : terms) {
|
|
||||||
String searchTerm = term.strip().toLowerCase() + "*";
|
|
||||||
for (var entry : fieldWeights.entrySet()) {
|
|
||||||
Query baseQuery = new WildcardQuery(new Term(entry.getKey(), searchTerm));
|
|
||||||
queryBuilder.add(new BoostQuery(baseQuery, entry.getValue()), BooleanClause.Occur.SHOULD);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BooleanQuery query = queryBuilder.build();
|
|
||||||
Path gymIndexDir = Path.of("gym-index");
|
|
||||||
try (
|
|
||||||
var reader = DirectoryReader.open(FSDirectory.open(gymIndexDir))
|
|
||||||
) {
|
|
||||||
IndexSearcher searcher = new IndexSearcher(reader);
|
|
||||||
List<GymResponse> results = new ArrayList<>(10);
|
|
||||||
TopDocs topDocs = searcher.search(query, 10, Sort.RELEVANCE, false);
|
|
||||||
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
|
|
||||||
Document doc = searcher.doc(scoreDoc.doc);
|
|
||||||
results.add(new GymResponse(doc));
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
} catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,106 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import nl.andrewlalis.gymboardsearch.dto.GymResponse;
|
||||||
|
import nl.andrewlalis.gymboardsearch.dto.UserResponse;
|
||||||
|
import org.apache.lucene.document.*;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Component that defines beans for the various indexes that this service
|
||||||
|
* supports. Beans are primarily constructed using the reusable "jdbc"
|
||||||
|
* components so that all index and search configuration is defined here.
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class IndexComponents {
|
||||||
|
@Bean
|
||||||
|
public JdbcConnectionSupplier jdbcConnectionSupplier() {
|
||||||
|
return () -> DriverManager.getConnection("jdbc:postgresql://localhost:5432/gymboard-api-dev", "gymboard-api-dev", "testpass");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public JdbcIndexGenerator userIndexGenerator(JdbcConnectionSupplier connectionSupplier) throws IOException {
|
||||||
|
return new JdbcIndexGenerator(
|
||||||
|
Path.of("user-index"),
|
||||||
|
connectionSupplier,
|
||||||
|
PlainQueryResultSetSupplier.fromResourceFile("/sql/select-users.sql"),
|
||||||
|
rs -> {
|
||||||
|
var doc = new Document();
|
||||||
|
doc.add(new StoredField("id", rs.getString("id")));
|
||||||
|
doc.add(new TextField("name", rs.getString("name"), Field.Store.YES));
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public QueryIndexSearcher<UserResponse> userIndexSearcher() {
|
||||||
|
return new QueryIndexSearcher<>(
|
||||||
|
UserResponse::new,
|
||||||
|
s -> new WeightedWildcardQueryBuilder()
|
||||||
|
.withField("name", 1f)
|
||||||
|
.build(s),
|
||||||
|
10,
|
||||||
|
Path.of("user-index")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public JdbcIndexGenerator gymIndexGenerator(JdbcConnectionSupplier connectionSupplier) throws IOException {
|
||||||
|
return new JdbcIndexGenerator(
|
||||||
|
Path.of("gym-index"),
|
||||||
|
connectionSupplier,
|
||||||
|
PlainQueryResultSetSupplier.fromResourceFile("/sql/select-gyms.sql"),
|
||||||
|
rs -> {
|
||||||
|
String shortName = rs.getString("short_name");
|
||||||
|
String displayName = rs.getString("display_name");
|
||||||
|
String cityShortName = rs.getString("city_short_name");
|
||||||
|
String cityName = rs.getString("city_name");
|
||||||
|
String countryCode = rs.getString("country_code");
|
||||||
|
String countryName = rs.getString("country_name");
|
||||||
|
String streetAddress = rs.getString("street_address");
|
||||||
|
BigDecimal latitude = rs.getBigDecimal("latitude");
|
||||||
|
BigDecimal longitude = rs.getBigDecimal("longitude");
|
||||||
|
String gymCompoundId = String.format("%s_%s_%s", countryCode, cityShortName, shortName);
|
||||||
|
|
||||||
|
Document doc = new Document();
|
||||||
|
doc.add(new StoredField("compound_id", gymCompoundId));
|
||||||
|
doc.add(new TextField("short_name", shortName, Field.Store.YES));
|
||||||
|
doc.add(new TextField("display_name", displayName, Field.Store.YES));
|
||||||
|
doc.add(new TextField("city_short_name", cityShortName, Field.Store.YES));
|
||||||
|
doc.add(new TextField("city_name", cityName, Field.Store.YES));
|
||||||
|
doc.add(new TextField("country_code", countryCode, Field.Store.YES));
|
||||||
|
doc.add(new TextField("country_name", countryName, Field.Store.YES));
|
||||||
|
doc.add(new TextField("street_address", streetAddress, Field.Store.YES));
|
||||||
|
doc.add(new DoublePoint("latitude_point", latitude.doubleValue()));
|
||||||
|
doc.add(new StoredField("latitude", latitude.doubleValue()));
|
||||||
|
doc.add(new DoublePoint("longitude_point", longitude.doubleValue()));
|
||||||
|
doc.add(new StoredField("longitude", longitude.doubleValue()));
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public QueryIndexSearcher<GymResponse> gymIndexSearcher() {
|
||||||
|
return new QueryIndexSearcher<>(
|
||||||
|
GymResponse::new,
|
||||||
|
s -> new WeightedWildcardQueryBuilder()
|
||||||
|
.withField("short_name", 3f)
|
||||||
|
.withField("display_name", 3f)
|
||||||
|
.withField("city_short_name", 1f)
|
||||||
|
.withField("city_name", 1f)
|
||||||
|
.withField("country_code", 0.25f)
|
||||||
|
.withField("country_name", 0.5f)
|
||||||
|
.withField("street_address", 0.1f)
|
||||||
|
.build(s),
|
||||||
|
10,
|
||||||
|
Path.of("gym-index")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,8 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
|
||||||
|
public interface JdbcConnectionSupplier {
|
||||||
|
Connection getConnection() throws SQLException;
|
||||||
|
}
|
|
@ -0,0 +1,67 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import org.apache.lucene.analysis.Analyzer;
|
||||||
|
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||||
|
import org.apache.lucene.index.IndexWriter;
|
||||||
|
import org.apache.lucene.index.IndexWriterConfig;
|
||||||
|
import org.apache.lucene.store.Directory;
|
||||||
|
import org.apache.lucene.store.FSDirectory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.util.FileSystemUtils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
|
||||||
|
public class JdbcIndexGenerator {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(JdbcIndexGenerator.class);
|
||||||
|
|
||||||
|
private final Path indexDir;
|
||||||
|
private final JdbcConnectionSupplier connectionSupplier;
|
||||||
|
private final JdbcResultSetSupplier resultSetSupplier;
|
||||||
|
private final JdbcResultDocumentMapper resultMapper;
|
||||||
|
|
||||||
|
public JdbcIndexGenerator(Path indexDir, JdbcConnectionSupplier connectionSupplier, JdbcResultSetSupplier resultSetSupplier, JdbcResultDocumentMapper resultMapper) {
|
||||||
|
this.indexDir = indexDir;
|
||||||
|
this.connectionSupplier = connectionSupplier;
|
||||||
|
this.resultSetSupplier = resultSetSupplier;
|
||||||
|
this.resultMapper = resultMapper;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void generate() {
|
||||||
|
log.info("Generating index at {}.", indexDir);
|
||||||
|
if (Files.exists(indexDir)) {
|
||||||
|
try {
|
||||||
|
FileSystemUtils.deleteRecursively(indexDir);
|
||||||
|
Files.createDirectory(indexDir);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Failed to reset index directory.", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try (
|
||||||
|
Connection conn = connectionSupplier.getConnection();
|
||||||
|
ResultSet rs = resultSetSupplier.supply(conn);
|
||||||
|
|
||||||
|
Analyzer analyzer = new StandardAnalyzer();
|
||||||
|
Directory luceneDir = FSDirectory.open(indexDir);
|
||||||
|
IndexWriter indexWriter = new IndexWriter(luceneDir, new IndexWriterConfig(analyzer))
|
||||||
|
) {
|
||||||
|
long count = 0;
|
||||||
|
while (rs.next()) {
|
||||||
|
try {
|
||||||
|
indexWriter.addDocument(resultMapper.map(rs));
|
||||||
|
count++;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("Failed to add document.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("Indexed {} entities.", count);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("Failed to prepare indexing components.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface JdbcResultDocumentMapper {
|
||||||
|
Document map(ResultSet rs) throws Exception;
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
|
||||||
|
public interface JdbcResultSetSupplier {
|
||||||
|
ResultSet supply(Connection conn) throws SQLException, IOException;
|
||||||
|
}
|
|
@ -0,0 +1,27 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import nl.andrewlalis.gymboardsearch.DbUtils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
|
||||||
|
public class PlainQueryResultSetSupplier implements JdbcResultSetSupplier {
|
||||||
|
private final String query;
|
||||||
|
|
||||||
|
public PlainQueryResultSetSupplier(String query) {
|
||||||
|
this.query = query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PlainQueryResultSetSupplier fromResourceFile(String resource) throws IOException {
|
||||||
|
return new PlainQueryResultSetSupplier(DbUtils.loadClasspathString(resource));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ResultSet supply(Connection conn) throws SQLException {
|
||||||
|
PreparedStatement stmt = conn.prepareStatement(query);
|
||||||
|
return stmt.executeQuery();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,53 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
import org.apache.lucene.index.DirectoryReader;
|
||||||
|
import org.apache.lucene.search.*;
|
||||||
|
import org.apache.lucene.store.FSDirectory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.function.Function;
|
||||||
|
|
||||||
|
public class QueryIndexSearcher<R> {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(QueryIndexSearcher.class);
|
||||||
|
|
||||||
|
private final Function<Document, R> mapper;
|
||||||
|
private final Function<String, Optional<Query>> querySupplier;
|
||||||
|
private final int maxResults;
|
||||||
|
private final Path indexDir;
|
||||||
|
|
||||||
|
public QueryIndexSearcher(Function<Document, R> mapper, Function<String, Optional<Query>> querySupplier, int maxResults, Path indexDir) {
|
||||||
|
this.mapper = mapper;
|
||||||
|
this.querySupplier = querySupplier;
|
||||||
|
this.maxResults = maxResults;
|
||||||
|
this.indexDir = indexDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<R> search(String rawQuery) {
|
||||||
|
Optional<Query> optionalQuery = querySupplier.apply(rawQuery);
|
||||||
|
if (optionalQuery.isEmpty()) return Collections.emptyList();
|
||||||
|
Query query = optionalQuery.get();
|
||||||
|
try (
|
||||||
|
var reader = DirectoryReader.open(FSDirectory.open(indexDir))
|
||||||
|
) {
|
||||||
|
IndexSearcher searcher = new IndexSearcher(reader);
|
||||||
|
List<R> results = new ArrayList<>(maxResults);
|
||||||
|
TopDocs topDocs = searcher.search(query, maxResults, Sort.RELEVANCE, false);
|
||||||
|
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
|
||||||
|
Document doc = searcher.doc(scoreDoc.doc);
|
||||||
|
results.add(mapper.apply(doc));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Could not search index.", e);
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,40 @@
|
||||||
|
package nl.andrewlalis.gymboardsearch.index;
|
||||||
|
|
||||||
|
import org.apache.lucene.index.Term;
|
||||||
|
import org.apache.lucene.search.*;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
|
public class WeightedWildcardQueryBuilder {
|
||||||
|
private final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
|
||||||
|
private final Map<String, Float> fieldWeights = new HashMap<>();
|
||||||
|
|
||||||
|
public WeightedWildcardQueryBuilder withField(String fieldName, float weight) {
|
||||||
|
fieldWeights.put(fieldName, weight);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public WeightedWildcardQueryBuilder customize(Consumer<BooleanQuery.Builder> customizer) {
|
||||||
|
customizer.accept(queryBuilder);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Optional<Query> build(String rawSearchQuery) {
|
||||||
|
if (rawSearchQuery == null || rawSearchQuery.isBlank()) return Optional.empty();
|
||||||
|
String[] terms = rawSearchQuery.toLowerCase().split("\\s+");
|
||||||
|
for (String term : terms) {
|
||||||
|
String searchTerm = term + "*";
|
||||||
|
for (var entry : fieldWeights.entrySet()) {
|
||||||
|
String fieldName = entry.getKey();
|
||||||
|
float weight = entry.getValue();
|
||||||
|
|
||||||
|
Query baseQuery = new WildcardQuery(new Term(fieldName, searchTerm));
|
||||||
|
queryBuilder.add(new BoostQuery(baseQuery, weight), BooleanClause.Occur.SHOULD);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Optional.of(queryBuilder.build());
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,7 @@
|
||||||
|
SELECT
|
||||||
|
u.id as id,
|
||||||
|
u.email as email,
|
||||||
|
u.name as name
|
||||||
|
FROM auth_user u
|
||||||
|
WHERE u.activated = TRUE
|
||||||
|
ORDER BY u.created_at;
|
Loading…
Reference in New Issue