From 4226200b433002de1fbe92ba3e7e2ffbbe632142 Mon Sep 17 00:00:00 2001 From: Gunnar Morling Date: Thu, 28 Dec 2023 11:44:58 +0100 Subject: =?UTF-8?q?=F0=9F=8F=86=20Initial=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/dev/morling/onebrc/CalculateAverage.java | 44 ++++++++++++ .../dev/morling/onebrc/CreateMeasurements.java | 83 ++++++++++++++++++++++ src/main/resources/.dontdelete | 0 src/test/resources/.dontdelete | 0 4 files changed, 127 insertions(+) create mode 100644 src/main/java/dev/morling/onebrc/CalculateAverage.java create mode 100644 src/main/java/dev/morling/onebrc/CreateMeasurements.java create mode 100644 src/main/resources/.dontdelete create mode 100644 src/test/resources/.dontdelete (limited to 'src') diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage.java b/src/main/java/dev/morling/onebrc/CalculateAverage.java new file mode 100644 index 0000000..7672d75 --- /dev/null +++ b/src/main/java/dev/morling/onebrc/CalculateAverage.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 The original authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.morling.onebrc; + +import static java.util.stream.Collectors.averagingDouble; +import static java.util.stream.Collectors.groupingBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +public class CalculateAverage { + + private static final String FILE = "./measurements.txt"; + + public static void main(String[] args) throws IOException { + Map measurements = Files.lines(Paths.get(FILE)) + .limit(10_000_000) + .map(l -> l.split(";")) + .collect(groupingBy(m -> m[0], averagingDouble(m -> Double.parseDouble(m[1])))); + + measurements = new TreeMap<>(measurements.entrySet() + .stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> Math.round(e.getValue() * 10.0) / 10.0))); + + System.out.println(measurements); + } +} diff --git a/src/main/java/dev/morling/onebrc/CreateMeasurements.java b/src/main/java/dev/morling/onebrc/CreateMeasurements.java new file mode 100644 index 0000000..583c9b1 --- /dev/null +++ b/src/main/java/dev/morling/onebrc/CreateMeasurements.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 The original authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.morling.onebrc; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +public class CreateMeasurements { + + private static final String FILE = "./measurements.txt"; + + private record WeatherStation(String id, double meanTemperature) { + double measurement() { + double m = ThreadLocalRandom.current().nextGaussian(meanTemperature, 10); + return Math.round(m * 10.0) / 10.0; + } + } + + public static void main(String[] args) throws Exception { + long start = System.currentTimeMillis(); + + if (args.length != 1) { + System.out.println("Usage: create_measurements.sh "); + System.exit(1); + } + + int size = 0; + try { + size = Integer.parseInt(args[0]); + } + catch (NumberFormatException e) { + System.out.println("Invalid value for "); + System.out.println("Usage: CreateMeasurements "); + System.exit(1); + } + + List stations = Arrays.asList( + new WeatherStation("auckland", 15.2), + new WeatherStation("concordia", -51.7), + new WeatherStation("lima", 19.2), + new WeatherStation("hamburg", 9.4), + new WeatherStation("hammerfest", 2.7), + new WeatherStation("maui", 15.2), + new WeatherStation("miami", 24.5), + new WeatherStation("nairobi", 17.8), + new WeatherStation("newdelhi", 25.3), + new WeatherStation("tokio", 15.8)); + + File measurements = new File(FILE); + try (FileOutputStream fos = new FileOutputStream(measurements); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos));) { + for (int i = 0; i < size; i++) { + if (i > 0 && i % 50_000_000 == 0) { + System.out.println("Wrote %,d measurements in %s ms".formatted(i, System.currentTimeMillis() - start)); + } + WeatherStation station = stations.get(ThreadLocalRandom.current().nextInt(stations.size())); + bw.write(station.id()); + bw.write(";" + station.measurement()); + bw.newLine(); + } + bw.flush(); + + System.out.println("Created file with %,d measurements in %s ms".formatted(size, System.currentTimeMillis() - start)); + } + } +} diff --git a/src/main/resources/.dontdelete b/src/main/resources/.dontdelete new file mode 100644 index 0000000..e69de29 diff --git a/src/test/resources/.dontdelete b/src/test/resources/.dontdelete new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3