automated download of systems and stations

used to populate database if empty

README.md

@@ -13,21 +13,11 @@ Its goal should be to know where you are ...
 
 ### Running
 
-1. download https://eddb.io/archive/v6/systems_populated.jsonl to `/home/vscode/systems_populated.jsonl`
+1. run backend server
-```shell
-# wget --compression=gzip -O /home/vscode/systems_populated.jsonl https://eddb.io/archive/v6/systems_populated.jsonl
-```
-
-2. download https://eddb.io/archive/v6/stations.jsonl to `/home/vscode/stations.jsonl`
-```shell
-# wget --compression=gzip -O /home/vscode/stations.jsonl https://eddb.io/archive/v6/stations.jsonl
-```
-
-3. run backend server
 ```shell
 # ./mvnw spring-boot:run
 ```
 
-4. run frontend server
+2. run frontend server
 ```shell
 ```

src/main/java/io/dietz/ed/companion/component/StationDeserializer.java

@@ -0,0 +1,63 @@
+package io.dietz.ed.companion.component;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.TimeZone;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+
+import io.dietz.ed.companion.models.eddb.Station;
+import io.dietz.ed.companion.models.eddb.StationType;
+import io.dietz.ed.companion.models.eddb.System;
+
+public class StationDeserializer extends JsonDeserializer<Station> {
+
+    @Override
+    public Station deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+        StationJson jsonStation = p.readValueAs(StationJson.class);
+
+        System system = new System(jsonStation.systemId);
+
+        Station station = new Station(jsonStation.id, jsonStation.name, system);
+        station.setType(new StationType(jsonStation.typeId, jsonStation.typeName));
+        station.setMaxLandingPadSize(jsonStation.maxLandingPadSize);
+        station.setDistanceToStar(jsonStation.distanceToStar);
+        station.setPlanetary(jsonStation.isPlanetary);
+        station.setUpdatedAt(LocalDateTime.ofInstant(Instant.ofEpochSecond(jsonStation.updatedAt), TimeZone.getDefault().toZoneId()));
+
+        return station;
+    }
+
+    private static class StationJson {
+
+        public long id;
+
+        public String name;
+
+        @JsonProperty("updated_at")
+        public long updatedAt;
+
+        @JsonProperty("system_id")
+        public long systemId;
+
+        @JsonProperty("type_id")
+        public long typeId;
+
+        @JsonProperty("type")
+        public String typeName;
+
+        @JsonProperty("max_landing_pad_size")
+        public String maxLandingPadSize;
+
+        @JsonProperty("distance_to_star")
+        public long distanceToStar;
+
+        @JsonProperty("is_planetary")
+        public boolean isPlanetary;
+    }
+}

src/main/java/io/dietz/ed/companion/component/SystemDeserializer.java

@@ -20,9 +20,9 @@ public class SystemDeserializer extends JsonDeserializer<System> {
         SystemJson jsonSystem = p.readValueAs(SystemJson.class);
 
         GeometryFactory gf = new GeometryFactory();
-        Point test = gf.createPoint(new Coordinate(jsonSystem.x, jsonSystem.y, jsonSystem.z));
+        Point systemPointInGalaxy = gf.createPoint(new Coordinate(jsonSystem.x, jsonSystem.y, jsonSystem.z));
 
-        return new System(jsonSystem.id, jsonSystem.name, test);
+        return new System(jsonSystem.id, jsonSystem.name, systemPointInGalaxy);
     }
 
     private static class SystemJson {

src/main/java/io/dietz/ed/companion/config/DatabaseConfiguration.java

@@ -2,7 +2,10 @@ package io.dietz.ed.companion.config;
 
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.time.LocalDate;
 import java.util.stream.Stream;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -12,9 +15,18 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.context.event.EventListener;
+import org.springframework.core.io.buffer.DataBuffer;
+import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
+import org.springframework.web.reactive.function.client.WebClient;
 
+import io.dietz.ed.companion.models.eddb.Station;
 import io.dietz.ed.companion.models.eddb.System;
+import io.dietz.ed.companion.repositories.StationRepository;
+import io.dietz.ed.companion.repositories.StationTypeRepository;
 import io.dietz.ed.companion.repositories.SystemRepository;
+import reactor.core.publisher.Flux;
+import reactor.netty.http.client.HttpClient;
 
 @Configuration
 public class DatabaseConfiguration {
@@ -25,10 +37,18 @@ public class DatabaseConfiguration {
     @Autowired
     SystemRepository systemRepository;
 
+    @Autowired
+    StationRepository stationRepository;
+
+    @Autowired
+    StationTypeRepository stationTypeRepository;
+
     @EventListener({ ContextRefreshedEvent.class })
-    public void onContextRefreshed() {
+    public void onContextRefreshed() throws IOException {
         if (systemRepository.count() == 0) {
-            try (Stream<String> stream = Files.lines(Paths.get("/home/vscode/systems_populated.jsonl"))) {
+            Path tmpFile = getEddbFile("systems_populated.jsonl");
+
+            try (Stream<String> stream = Files.lines(tmpFile)) {
                 stream.forEach(systemString -> {
                     try {
                         System system = objectMapper.readValue(systemString, System.class);
@@ -43,6 +63,64 @@ public class DatabaseConfiguration {
                 e.printStackTrace();
             }
         }
+
+        if (stationRepository.count() == 0) {
+            Path tmpFile = getEddbFile("stations.jsonl");
+
+            try (Stream<String> stream = Files.lines(tmpFile)) {
+                stream.forEach(stationString -> {
+                    try {
+                        Station station = objectMapper.readValue(stationString, Station.class);
+
+                        // check if StationType was already persisted
+                        if (stationTypeRepository.findById(station.getType().getId()).isEmpty()) {
+                            stationTypeRepository.saveAndFlush(station.getType());
+                        }
+
+                        // check if System is existing
+                        if (systemRepository.findById(station.getSystem().getId()).isEmpty()) {
+                            return;
+                        }
+
+                        // skip "Fleet Carrier"s
+                        if (station.getType().getId() == 24) {
+                            java.lang.System.out.print("skipped ");
+                        } else {
+                            stationRepository.save(station);
+                        }
+                        java.lang.System.out.println(station);
+                    } catch (JsonProcessingException e) {
+                        e.printStackTrace();
+                    }
+                });
+                stationRepository.flush();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    private Path getEddbFile(String file) throws IOException {
+        String tmpDir = java.lang.System.getProperty("java.io.tmpdir");
+        LocalDate date = LocalDate.now();
+        Path tmpFile = Paths.get(tmpDir, date + file);
+
+        if (Files.exists(tmpFile)) {
+            return tmpFile;
+        }
+
+        java.lang.System.out.println(tmpFile.toString() + " for " + file);
+
+        Flux<DataBuffer> dataBufferFlux = getWebClient().get().uri("/" + file).retrieve().bodyToFlux(DataBuffer.class);
+        DataBufferUtils.write(dataBufferFlux, tmpFile, StandardOpenOption.CREATE).block();
+
+        return tmpFile;
+    }
+
+    private WebClient getWebClient() {
+        HttpClient httpClient = HttpClient.create().compress(true);
+        return WebClient.builder().clientConnector(new ReactorClientHttpConnector(httpClient))
+                .baseUrl("https://eddb.io/archive/v6").build();
     }
 
 }

src/main/java/io/dietz/ed/companion/domain/Market.java

@@ -11,7 +11,7 @@ public class Market {
 
     @Override
     public String toString() {
-        return systemName + " - " + stationName + " (" + marketId + ")";
+        return systemName + " - " + stationName + " (" + marketId + ") (" + commodities.size() + " commodities)";
     }    
 
 }

src/main/java/io/dietz/ed/companion/models/eddb/Market.java

@@ -0,0 +1,14 @@
+package io.dietz.ed.companion.models.eddb;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.OneToOne;
+
+@Entity
+public class Market {
+    @Id
+    protected long id;
+
+    @OneToOne(mappedBy = "market")
+    protected Station station;
+}

src/main/java/io/dietz/ed/companion/models/eddb/Station.java

@@ -1,19 +1,117 @@
 package io.dietz.ed.companion.models.eddb;
 
+import java.time.LocalDateTime;
+
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
+import javax.persistence.OneToOne;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import io.dietz.ed.companion.component.StationDeserializer;
 
 @Entity
+@JsonDeserialize(using = StationDeserializer.class)
 public class Station {
     @Id
     protected long id;
 
     protected String name;
 
+    protected LocalDateTime updatedAt;
+
+    protected String maxLandingPadSize;
+
+    protected long distanceToStar;
+
+    protected boolean isPlanetary;
+
+    protected boolean hasBlackmarket = false;
+    protected boolean hasMarket = false;
+    protected boolean hasRefuel = false;
+    protected boolean hasRepair = false;
+    protected boolean hasRearm = false;
+    protected boolean hasOutfitting = false;
+    protected boolean hasShipyard = false;
+    protected boolean hasDocking = false;
+    protected boolean hasCommodities = false;
+    protected boolean hasMaterialTrader = false;
+    protected boolean hasTechnologyBroker = false;
+    protected boolean hasCarrierVendor = false;
+    protected boolean hasCarrierAdministration = false;
+    protected boolean hasInterstellarFactors = false;
+    protected boolean hasUniversalCartographics = false;
+
     @ManyToOne
     @JoinColumn(name = "system_id")
     protected System system;
 
+    @OneToOne
+    @JoinColumn(name = "market_id")
+    protected Market market;
+
+    @OneToOne
+    @JoinColumn(name = "type_id")
+    protected StationType type;
+
+    public Station() {
+    }
+
+    public Station(long id, String name, System system) {
+        this.id = id;
+        this.name = name;
+        this.system = system;
+    }
+
+    public System getSystem() {
+        return system;
+    }
+
+    public StationType getType() {
+        return type;
+    }
+
+    public void setType(StationType type) {
+        this.type = type;
+    }
+
+    public LocalDateTime getUpdatedAt() {
+        return updatedAt;
+    }
+
+    public void setUpdatedAt(LocalDateTime updatedAt) {
+        this.updatedAt = updatedAt;
+    }
+
+    public String getMaxLandingPadSize() {
+        return maxLandingPadSize;
+    }
+
+    public void setMaxLandingPadSize(String maxLandingPadSize) {
+        this.maxLandingPadSize = maxLandingPadSize;
+    }
+
+    public long getDistanceToStar() {
+        return distanceToStar;
+    }
+
+    public void setDistanceToStar(long distanceToStar) {
+        this.distanceToStar = distanceToStar;
+    }
+
+    public boolean isPlanetary() {
+        return isPlanetary;
+    }
+
+    public void setPlanetary(boolean isPlanetary) {
+        this.isPlanetary = isPlanetary;
+    }
+
+    @Override
+    public String toString() {
+        return "Station [name=" + name + "]";
+    }
+
 }

src/main/java/io/dietz/ed/companion/models/eddb/StationType.java

@@ -0,0 +1,25 @@
+package io.dietz.ed.companion.models.eddb;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+
+@Entity
+public class StationType {
+    @Id
+    protected long id;
+
+    protected String name;
+
+    public StationType() {
+    }
+
+    public StationType(long id, String name) {
+        this.id = id;
+        this.name = name;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+}

src/main/java/io/dietz/ed/companion/models/eddb/System.java

@@ -21,11 +21,24 @@ public class System {
 
     public System() {
     }
-    
+
+    public System(long id) {
+        this.id = id;
+    }
+
     public System(long id, String name, Point location) {
         this.id = id;
         this.name = name;
         this.location = location;
     }
 
+    public long getId() {
+        return id;
+    }
+
+    @Override
+    public String toString() {
+        return "System [name=" + name + "]";
+    }
+
 }

src/main/java/io/dietz/ed/companion/repositories/StationRepository.java

@@ -0,0 +1,9 @@
+package io.dietz.ed.companion.repositories;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import io.dietz.ed.companion.models.eddb.Station;
+
+public interface StationRepository extends JpaRepository<Station, Long> {
+
+}

src/main/java/io/dietz/ed/companion/repositories/StationTypeRepository.java

@@ -0,0 +1,9 @@
+package io.dietz.ed.companion.repositories;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import io.dietz.ed.companion.models.eddb.StationType;
+
+public interface StationTypeRepository extends JpaRepository<StationType, Long> {
+
+}