From f5c79ed5f29b4dd6bcdc1d4480cb189086ae69fc Mon Sep 17 00:00:00 2001 From: Steffen Dietz Date: Fri, 7 May 2021 21:21:57 +0000 Subject: [PATCH] automated download of systems and stations used to populate database if empty --- README.md | 14 +-- .../component/StationDeserializer.java | 63 ++++++++++++ .../component/SystemDeserializer.java | 4 +- .../config/DatabaseConfiguration.java | 82 +++++++++++++++- .../io/dietz/ed/companion/domain/Market.java | 2 +- .../ed/companion/models/eddb/Market.java | 14 +++ .../ed/companion/models/eddb/Station.java | 98 +++++++++++++++++++ .../ed/companion/models/eddb/StationType.java | 25 +++++ .../ed/companion/models/eddb/System.java | 15 ++- .../repositories/StationRepository.java | 9 ++ .../repositories/StationTypeRepository.java | 9 ++ 11 files changed, 317 insertions(+), 18 deletions(-) create mode 100644 src/main/java/io/dietz/ed/companion/component/StationDeserializer.java create mode 100644 src/main/java/io/dietz/ed/companion/models/eddb/Market.java create mode 100644 src/main/java/io/dietz/ed/companion/models/eddb/StationType.java create mode 100644 src/main/java/io/dietz/ed/companion/repositories/StationRepository.java create mode 100644 src/main/java/io/dietz/ed/companion/repositories/StationTypeRepository.java diff --git a/README.md b/README.md index 61004e0..f0c4c44 100644 --- a/README.md +++ b/README.md @@ -13,21 +13,11 @@ Its goal should be to know where you are ... ### Running -1. download https://eddb.io/archive/v6/systems_populated.jsonl to `/home/vscode/systems_populated.jsonl` -```shell -# wget --compression=gzip -O /home/vscode/systems_populated.jsonl https://eddb.io/archive/v6/systems_populated.jsonl -``` - -2. download https://eddb.io/archive/v6/stations.jsonl to `/home/vscode/stations.jsonl` -```shell -# wget --compression=gzip -O /home/vscode/stations.jsonl https://eddb.io/archive/v6/stations.jsonl -``` - -3. run backend server +1. run backend server ```shell # ./mvnw spring-boot:run ``` -4. run frontend server +2. run frontend server ```shell ``` \ No newline at end of file diff --git a/src/main/java/io/dietz/ed/companion/component/StationDeserializer.java b/src/main/java/io/dietz/ed/companion/component/StationDeserializer.java new file mode 100644 index 0000000..20f4b64 --- /dev/null +++ b/src/main/java/io/dietz/ed/companion/component/StationDeserializer.java @@ -0,0 +1,63 @@ +package io.dietz.ed.companion.component; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.TimeZone; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import io.dietz.ed.companion.models.eddb.Station; +import io.dietz.ed.companion.models.eddb.StationType; +import io.dietz.ed.companion.models.eddb.System; + +public class StationDeserializer extends JsonDeserializer { + + @Override + public Station deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { + StationJson jsonStation = p.readValueAs(StationJson.class); + + System system = new System(jsonStation.systemId); + + Station station = new Station(jsonStation.id, jsonStation.name, system); + station.setType(new StationType(jsonStation.typeId, jsonStation.typeName)); + station.setMaxLandingPadSize(jsonStation.maxLandingPadSize); + station.setDistanceToStar(jsonStation.distanceToStar); + station.setPlanetary(jsonStation.isPlanetary); + station.setUpdatedAt(LocalDateTime.ofInstant(Instant.ofEpochSecond(jsonStation.updatedAt), TimeZone.getDefault().toZoneId())); + + return station; + } + + private static class StationJson { + + public long id; + + public String name; + + @JsonProperty("updated_at") + public long updatedAt; + + @JsonProperty("system_id") + public long systemId; + + @JsonProperty("type_id") + public long typeId; + + @JsonProperty("type") + public String typeName; + + @JsonProperty("max_landing_pad_size") + public String maxLandingPadSize; + + @JsonProperty("distance_to_star") + public long distanceToStar; + + @JsonProperty("is_planetary") + public boolean isPlanetary; + } +} diff --git a/src/main/java/io/dietz/ed/companion/component/SystemDeserializer.java b/src/main/java/io/dietz/ed/companion/component/SystemDeserializer.java index c0db3c4..4b354e7 100644 --- a/src/main/java/io/dietz/ed/companion/component/SystemDeserializer.java +++ b/src/main/java/io/dietz/ed/companion/component/SystemDeserializer.java @@ -20,9 +20,9 @@ public class SystemDeserializer extends JsonDeserializer { SystemJson jsonSystem = p.readValueAs(SystemJson.class); GeometryFactory gf = new GeometryFactory(); - Point test = gf.createPoint(new Coordinate(jsonSystem.x, jsonSystem.y, jsonSystem.z)); + Point systemPointInGalaxy = gf.createPoint(new Coordinate(jsonSystem.x, jsonSystem.y, jsonSystem.z)); - return new System(jsonSystem.id, jsonSystem.name, test); + return new System(jsonSystem.id, jsonSystem.name, systemPointInGalaxy); } private static class SystemJson { diff --git a/src/main/java/io/dietz/ed/companion/config/DatabaseConfiguration.java b/src/main/java/io/dietz/ed/companion/config/DatabaseConfiguration.java index 4aa0bd9..695cca1 100644 --- a/src/main/java/io/dietz/ed/companion/config/DatabaseConfiguration.java +++ b/src/main/java/io/dietz/ed/companion/config/DatabaseConfiguration.java @@ -2,7 +2,10 @@ package io.dietz.ed.companion.config; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.time.LocalDate; import java.util.stream.Stream; import com.fasterxml.jackson.core.JsonProcessingException; @@ -12,9 +15,18 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferUtils; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; +import io.dietz.ed.companion.models.eddb.Station; import io.dietz.ed.companion.models.eddb.System; +import io.dietz.ed.companion.repositories.StationRepository; +import io.dietz.ed.companion.repositories.StationTypeRepository; import io.dietz.ed.companion.repositories.SystemRepository; +import reactor.core.publisher.Flux; +import reactor.netty.http.client.HttpClient; @Configuration public class DatabaseConfiguration { @@ -25,10 +37,18 @@ public class DatabaseConfiguration { @Autowired SystemRepository systemRepository; + @Autowired + StationRepository stationRepository; + + @Autowired + StationTypeRepository stationTypeRepository; + @EventListener({ ContextRefreshedEvent.class }) - public void onContextRefreshed() { + public void onContextRefreshed() throws IOException { if (systemRepository.count() == 0) { - try (Stream stream = Files.lines(Paths.get("/home/vscode/systems_populated.jsonl"))) { + Path tmpFile = getEddbFile("systems_populated.jsonl"); + + try (Stream stream = Files.lines(tmpFile)) { stream.forEach(systemString -> { try { System system = objectMapper.readValue(systemString, System.class); @@ -43,6 +63,64 @@ public class DatabaseConfiguration { e.printStackTrace(); } } + + if (stationRepository.count() == 0) { + Path tmpFile = getEddbFile("stations.jsonl"); + + try (Stream stream = Files.lines(tmpFile)) { + stream.forEach(stationString -> { + try { + Station station = objectMapper.readValue(stationString, Station.class); + + // check if StationType was already persisted + if (stationTypeRepository.findById(station.getType().getId()).isEmpty()) { + stationTypeRepository.saveAndFlush(station.getType()); + } + + // check if System is existing + if (systemRepository.findById(station.getSystem().getId()).isEmpty()) { + return; + } + + // skip "Fleet Carrier"s + if (station.getType().getId() == 24) { + java.lang.System.out.print("skipped "); + } else { + stationRepository.save(station); + } + java.lang.System.out.println(station); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + }); + stationRepository.flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private Path getEddbFile(String file) throws IOException { + String tmpDir = java.lang.System.getProperty("java.io.tmpdir"); + LocalDate date = LocalDate.now(); + Path tmpFile = Paths.get(tmpDir, date + file); + + if (Files.exists(tmpFile)) { + return tmpFile; + } + + java.lang.System.out.println(tmpFile.toString() + " for " + file); + + Flux dataBufferFlux = getWebClient().get().uri("/" + file).retrieve().bodyToFlux(DataBuffer.class); + DataBufferUtils.write(dataBufferFlux, tmpFile, StandardOpenOption.CREATE).block(); + + return tmpFile; + } + + private WebClient getWebClient() { + HttpClient httpClient = HttpClient.create().compress(true); + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(httpClient)) + .baseUrl("https://eddb.io/archive/v6").build(); } } \ No newline at end of file diff --git a/src/main/java/io/dietz/ed/companion/domain/Market.java b/src/main/java/io/dietz/ed/companion/domain/Market.java index 29a8554..9d5d73e 100644 --- a/src/main/java/io/dietz/ed/companion/domain/Market.java +++ b/src/main/java/io/dietz/ed/companion/domain/Market.java @@ -11,7 +11,7 @@ public class Market { @Override public String toString() { - return systemName + " - " + stationName + " (" + marketId + ")"; + return systemName + " - " + stationName + " (" + marketId + ") (" + commodities.size() + " commodities)"; } } \ No newline at end of file diff --git a/src/main/java/io/dietz/ed/companion/models/eddb/Market.java b/src/main/java/io/dietz/ed/companion/models/eddb/Market.java new file mode 100644 index 0000000..bd37823 --- /dev/null +++ b/src/main/java/io/dietz/ed/companion/models/eddb/Market.java @@ -0,0 +1,14 @@ +package io.dietz.ed.companion.models.eddb; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.OneToOne; + +@Entity +public class Market { + @Id + protected long id; + + @OneToOne(mappedBy = "market") + protected Station station; +} diff --git a/src/main/java/io/dietz/ed/companion/models/eddb/Station.java b/src/main/java/io/dietz/ed/companion/models/eddb/Station.java index d0260be..c246b3a 100644 --- a/src/main/java/io/dietz/ed/companion/models/eddb/Station.java +++ b/src/main/java/io/dietz/ed/companion/models/eddb/Station.java @@ -1,19 +1,117 @@ package io.dietz.ed.companion.models.eddb; +import java.time.LocalDateTime; + import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; +import javax.persistence.OneToOne; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import io.dietz.ed.companion.component.StationDeserializer; @Entity +@JsonDeserialize(using = StationDeserializer.class) public class Station { @Id protected long id; protected String name; + protected LocalDateTime updatedAt; + + protected String maxLandingPadSize; + + protected long distanceToStar; + + protected boolean isPlanetary; + + protected boolean hasBlackmarket = false; + protected boolean hasMarket = false; + protected boolean hasRefuel = false; + protected boolean hasRepair = false; + protected boolean hasRearm = false; + protected boolean hasOutfitting = false; + protected boolean hasShipyard = false; + protected boolean hasDocking = false; + protected boolean hasCommodities = false; + protected boolean hasMaterialTrader = false; + protected boolean hasTechnologyBroker = false; + protected boolean hasCarrierVendor = false; + protected boolean hasCarrierAdministration = false; + protected boolean hasInterstellarFactors = false; + protected boolean hasUniversalCartographics = false; + @ManyToOne @JoinColumn(name = "system_id") protected System system; + @OneToOne + @JoinColumn(name = "market_id") + protected Market market; + + @OneToOne + @JoinColumn(name = "type_id") + protected StationType type; + + public Station() { + } + + public Station(long id, String name, System system) { + this.id = id; + this.name = name; + this.system = system; + } + + public System getSystem() { + return system; + } + + public StationType getType() { + return type; + } + + public void setType(StationType type) { + this.type = type; + } + + public LocalDateTime getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(LocalDateTime updatedAt) { + this.updatedAt = updatedAt; + } + + public String getMaxLandingPadSize() { + return maxLandingPadSize; + } + + public void setMaxLandingPadSize(String maxLandingPadSize) { + this.maxLandingPadSize = maxLandingPadSize; + } + + public long getDistanceToStar() { + return distanceToStar; + } + + public void setDistanceToStar(long distanceToStar) { + this.distanceToStar = distanceToStar; + } + + public boolean isPlanetary() { + return isPlanetary; + } + + public void setPlanetary(boolean isPlanetary) { + this.isPlanetary = isPlanetary; + } + + @Override + public String toString() { + return "Station [name=" + name + "]"; + } + } \ No newline at end of file diff --git a/src/main/java/io/dietz/ed/companion/models/eddb/StationType.java b/src/main/java/io/dietz/ed/companion/models/eddb/StationType.java new file mode 100644 index 0000000..a70f4fb --- /dev/null +++ b/src/main/java/io/dietz/ed/companion/models/eddb/StationType.java @@ -0,0 +1,25 @@ +package io.dietz.ed.companion.models.eddb; + +import javax.persistence.Entity; +import javax.persistence.Id; + +@Entity +public class StationType { + @Id + protected long id; + + protected String name; + + public StationType() { + } + + public StationType(long id, String name) { + this.id = id; + this.name = name; + } + + public long getId() { + return id; + } + +} diff --git a/src/main/java/io/dietz/ed/companion/models/eddb/System.java b/src/main/java/io/dietz/ed/companion/models/eddb/System.java index b0ed231..94491a8 100644 --- a/src/main/java/io/dietz/ed/companion/models/eddb/System.java +++ b/src/main/java/io/dietz/ed/companion/models/eddb/System.java @@ -21,11 +21,24 @@ public class System { public System() { } - + + public System(long id) { + this.id = id; + } + public System(long id, String name, Point location) { this.id = id; this.name = name; this.location = location; } + public long getId() { + return id; + } + + @Override + public String toString() { + return "System [name=" + name + "]"; + } + } \ No newline at end of file diff --git a/src/main/java/io/dietz/ed/companion/repositories/StationRepository.java b/src/main/java/io/dietz/ed/companion/repositories/StationRepository.java new file mode 100644 index 0000000..842da81 --- /dev/null +++ b/src/main/java/io/dietz/ed/companion/repositories/StationRepository.java @@ -0,0 +1,9 @@ +package io.dietz.ed.companion.repositories; + +import org.springframework.data.jpa.repository.JpaRepository; + +import io.dietz.ed.companion.models.eddb.Station; + +public interface StationRepository extends JpaRepository { + +} diff --git a/src/main/java/io/dietz/ed/companion/repositories/StationTypeRepository.java b/src/main/java/io/dietz/ed/companion/repositories/StationTypeRepository.java new file mode 100644 index 0000000..7ab0230 --- /dev/null +++ b/src/main/java/io/dietz/ed/companion/repositories/StationTypeRepository.java @@ -0,0 +1,9 @@ +package io.dietz.ed.companion.repositories; + +import org.springframework.data.jpa.repository.JpaRepository; + +import io.dietz.ed.companion.models.eddb.StationType; + +public interface StationTypeRepository extends JpaRepository { + +}