aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGunnar Morling <gunnar.morling@googlemail.com>2024-01-12 21:21:40 +0100
committerGunnar Morling <gunnar.morling@googlemail.com>2024-01-12 21:21:40 +0100
commit4cfb5449e92728a2a4fb2478378819808ab7ed59 (patch)
tree7c83126dd5c867c9e55ee488040ad416a69f22ae
parent53776398dcf0d106826ebf19c9b2d807758327bd (diff)
Leaderboard, formatting
-rw-r--r--README.md16
-rwxr-xr-x[-rw-r--r--]calculate_average_ianopolous.sh0
-rw-r--r--src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java428
3 files changed, 223 insertions, 221 deletions
diff --git a/README.md b/README.md
index a2276f6..489f305 100644
--- a/README.md
+++ b/README.md
@@ -42,8 +42,8 @@ These are the results from running all entries into the challenge on eight cores
| # | Result (m:s.ms) | Implementation | JDK | Submitter | Notes |
|---|-----------------|--------------------|-----|---------------|-----------|
| 1 | 00:02.575 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_merykittyunsafe.java)| 21.0.1-open | [Quan Anh Mai](https://github.com/merykitty) | Quan Anh Mai's implementation, using `Unsafe` |
-| 2 | 00:02.904 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_royvanrijn.java)| 21.0.1-graal | [Roy van Rijn](https://github.com/royvanrijn) | GraalVM native binary |
-| 3 | 00:03.044 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_thomaswue.java)| 21.0.1-graal | [Thomas Wuerthinger](https://github.com/thomaswue), [Quan Anh Mai](https://github.com/merykitty), [Alfonso² Peterssen](https://github.com/mukel) | GraalVM native binary |
+| 2 | 00:02.708 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_thomaswue.java)| 21.0.1-graal | [Thomas Wuerthinger](https://github.com/thomaswue), [Quan Anh Mai](https://github.com/merykitty), [Alfonso² Peterssen](https://github.com/mukel) | GraalVM native binary |
+| 3 | 00:02.855 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_royvanrijn.java)| 21.0.1-graal | [Roy van Rijn](https://github.com/royvanrijn) | GraalVM native binary |
| | 00:03.258 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_merykitty.java)| 21.0.1-open | [Quan Anh Mai](https://github.com/merykitty) | |
| | 00:03.321 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_artsiomkorzun.java)| 21.0.1-graal | [Artsiom Korzun](https://github.com/artsiomkorzun) | |
| | 00:03.539 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_mtopolnik.java)| 21.0.1-graal | [Marko Topolnik](https://github.com/mtopolnik) | |
@@ -51,9 +51,11 @@ These are the results from running all entries into the challenge on eight cores
| | 00:04.823 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_JamalMulla.java)| 21.0.1-graal | [Jamal Mulla](https://github.com/JamalMulla) | |
| | 00:04.959 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_yavuztas.java)| 21.0.1-graal | [Yavuz Tas](https://github.com/yavuztas) | |
| | 00:04.996 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ebarlas.java)| 21.0.1-graal | [Elliot Barlas](https://github.com/ebarlas) | |
+| | 00:05.218 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_iziamos.java)| 21.0.1-open | [John Ziamos](https://github.com/iziamos) | |
| | 00:05.478 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_obourgain.java)| 21.0.1-open | [Olivier Bourgain](https://github.com/obourgain) | |
| | 00:05.979 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_spullara.java)| 21.0.1-graal | [Sam Pullara](https://github.com/spullara) | |
| | 00:06.166 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_isolgpus.java)| 21.0.1-open | [Jamie Stansfield](https://github.com/isolgpus) | |
+| | 00:06.576 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_asun.java)| 21.0.1-open | [Alan Sun](https://github.com/asun) | |
| | 00:06.715 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_algirdasrascius.java)| 21.0.1-open | [Algirdas Raščius](https://github.com/algirdasrascius) | |
| | 00:07.730 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_jotschi.java)| 21.0.1-open | [Johannes Schüth](https://github.com/jotschi) | |
| | 00:07.843 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_zerninv.java)| 21.0.1-open | [zerninv](https://github.com/zerninv) | |
@@ -65,25 +67,25 @@ These are the results from running all entries into the challenge on eight cores
| | 00:08.517 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ags313.java)| 21.0.1-graal | [ags](https://github.com/ags313) | |
| | 00:08.892 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_fatroom.java)| 21.0.1-open | [Roman Romanchuk](https://github.com/fatroom) | |
| | 00:09.020 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_yemreinci.java)| 21.0.1-open | [yemreinci](https://github.com/yemreinci) | |
-| | 00:09.046 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_iziamos.java)| 21.0.1-open | [John Ziamos](https://github.com/iziamos) | |
| | 00:09.071 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_gabrielreid.java)| 21.0.1-open | [Gabriel Reid](https://github.com/gabrielreid) | |
| | 00:09.117 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_kuduwa-keshavram.java)| 21.0.1-graal | [Keshavram Kuduwa](https://github.com/kuduwa-keshavram) | |
| | 00:09.352 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_filiphr.java)| 21.0.1-graal | [Filip Hrisafov](https://github.com/filiphr) | |
| | 00:09.867 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ricardopieper.java)| 21.0.1-graal | [Ricardo Pieper](https://github.com/ricardopieper) | |
| | 00:10.127 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_artpar.java)| 21.0.1-open | [Parth Mudgal](https://github.com/artpar) | |
+| | 00:10.553 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_C5H12O5.java)| 21.0.1-graal | [Xylitol](https://github.com/C5H12O5) | |
| | 00:10.949 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_roman-r-m.java)| 21.0.1-graal | [Roman Musin](https://github.com/roman-r-m) | |
| | 00:10.473 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_raipc.java)| 21.0.1-open | [Anton Rybochkin](https://github.com/raipc) | |
-| | 00:10.642 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_asun.java)| 21.0.1-open | [Alan Sun](https://github.com/asun) | |
| | 00:11.119 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_lawrey.java)| 21.0.1-open | [lawrey](https://github.com/lawrey) | |
| | 00:11.167 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_palmr.java)| 21.0.1-open | [Nick Palmer](https://github.com/palmr) | |
+| | 00:11.405 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_imrafaelmerino.java)| 21.0.1-graal | [Rafael Merino García](https://github.com/imrafaelmerino) | |
| | 00:11.805 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_coolmineman.java)| 21.0.1-graal | [Cool_Mineman](https://github.com/coolmineman) | |
| | 00:11.934 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_arjenvaneerde.java)| 21.0.1-open | [arjenvaneerde](https://github.com/arjenvaneerde) | |
| | 00:11.987 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_flippingbits.java)| 21.0.1-graal | [Stefan Sprenger](https://github.com/flippingbits) | |
| | 00:12.220 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_richardstartin.java)| 21.0.1-open | [Richard Startin](https://github.com/richardstartin) | |
-| | 00:12.471 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_SamuelYvon.java)| 21.0.1-graal | [Samuel Yvon](https://github.com/SamuelYvon) | |
-| | 00:12.557 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_imrafaelmerino.java)| 21.0.1-graal | [Rafael Merino García](https://github.com/imrafaelmerino) | |
+| | 00:12.495 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_SamuelYvon.java)| 21.0.1-graal | [Samuel Yvon](https://github.com/SamuelYvon) | GraalVM native binary |
| | 00:12.565 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_japplis.java)| 21.0.1-open | [Anthony Goubard](https://github.com/japplis) | |
| | 00:12.568 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_MeanderingProgrammer.java)| 21.0.1-graal | [Vlad](https://github.com/MeanderingProgrammer) | |
+| | 00:13.013 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_thanhtrinity.java)| 21.0.1-graal | [Thanh Duong](https://github.com/thanhtrinity) | |
| | 00:13.148 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_charlibot.java)| 21.0.1-open | [Charlie Evans](https://github.com/charlibot) | |
| | 00:13.623 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_phd3.java)| 21.0.1-open | [Pratham](https://github.com/phd3) | |
| | 00:13.817 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_entangled90.java)| 21.0.1-open | [Carlo](https://github.com/entangled90) | |
@@ -91,13 +93,13 @@ These are the results from running all entries into the challenge on eight cores
| | 00:14.867 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_berry120.java)| 21.0.1-open | [Michael Berry](https://github.com/berry120) | |
| | 00:15.662 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_semotpan.java)| 21.0.1-open | [Serghei Motpan](https://github.com/semotpan) | |
| | 00:15.752 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_jbachorik.java)| 21.0.1-tem | [Jaroslav Bachorik](https://github.com/jbachorik) | |
+| | 00:16.379 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ianopolous.java)| 21.0.1-open | [Dr Ian Preston](https://github.com/ianopolous) | |
| | 00:17.490 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_kgeri.java)| 21.0.1-open | [Gergely Kiss](https://github.com/kgeri) | |
| | 00:17.815 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_hallvard.java)| 21.0.1-open | [Hallvard Trætteberg](https://github.com/hallvard) | |
| | 00:18.251 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_seijikun.java)| 21.0.1-graal | [Markus Ebner](https://github.com/seijikun) | |
| | 00:18.448 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_moysesb.java)| 21.0.1-open | [Moysés Borges Furtado](https://github.com/moysesb) | |
| | 00:18.771 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_davecom.java)| 21.0.1-graal | [David Kopec](https://github.com/davecom) | |
| | 00:19.357 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_truelive.java)| 21.0.1-graalce | [Roman Schweitzer](https://github.com/truelive) | |
-| | 00:20.095 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_C5H12O5.java)| 21.0.1-graal | [Xylitol](https://github.com/C5H12O5) | |
| | 00:21.989 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_couragelee.java)| 21.0.1-open | [couragelee](https://github.com/couragelee) | |
| | 00:22.457 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_rby.java)| 21.0.1-open | [Ramzi Ben Yahya](https://github.com/rby) | |
| | 00:24.528 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_gnabyl.java)| 21.0.1-open | [Bang NGUYEN](https://github.com/gnabyl) | |
diff --git a/calculate_average_ianopolous.sh b/calculate_average_ianopolous.sh
index 5f0f23d..5f0f23d 100644..100755
--- a/calculate_average_ianopolous.sh
+++ b/calculate_average_ianopolous.sh
diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java b/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java
index 8d261ef..9d0027c 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java
@@ -14,217 +14,217 @@
* limitations under the License.
*/
- package dev.morling.onebrc;
-
- import java.io.IOException;
- import java.lang.foreign.Arena;
- import java.lang.reflect.Field;
- import java.nio.channels.FileChannel;
- import java.nio.channels.FileChannel.MapMode;
- import java.nio.charset.StandardCharsets;
- import java.nio.file.Path;
- import java.nio.file.StandardOpenOption;
- import java.util.TreeMap;
- import sun.misc.Unsafe;
-
- public class CalculateAverage_abeobk {
- private static final String FILE = "./measurements.txt";
- private static final int BUCKET_SIZE = 1 << 16;
- private static final int BUCKET_MASK = BUCKET_SIZE - 1;
- private static final int MAX_STR_LEN = 100;
- private static final Unsafe UNSAFE = initUnsafe();
- private static final long[] HASH_MASKS = new long[]{
- 0x0L,
- 0xffL,
- 0xffffL,
- 0xffffffL,
- 0xffffffffL,
- 0xffffffffffL,
- 0xffffffffffffL,
- 0xffffffffffffffL,
- 0xffffffffffffffffL, };
-
- private static Unsafe initUnsafe() {
- try {
- Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
- theUnsafe.setAccessible(true);
- return (Unsafe) theUnsafe.get(Unsafe.class);
- }
- catch (Exception ex) {
- throw new RuntimeException();
- }
- }
-
- // stat
- private static class Stat {
- private int min;
- private int max;
- private long sum;
- private int count;
-
- Stat(int v) {
- sum = min = max = v;
- count = 1;
- }
-
- void add(int val) {
- min = Math.min(val, min);
- max = Math.max(val, max);
- sum += val;
- count++;
- }
-
- void merge(Stat other) {
- min = Math.min(other.min, min);
- max = Math.max(other.max, max);
- sum += other.sum;
- count += other.count;
- }
-
- public String toString() {
- return String.format("%.1f/%.1f/%.1f", min * 0.1, sum * 0.1 / count, max * 0.1);
- }
- }
-
- static class Node {
- long addr;
- int keylen;
- int hash;
- long[] buf = new long[13];
- Stat stat;
-
- String key() {
- byte[] buf = new byte[MAX_STR_LEN];
- UNSAFE.copyMemory(null, addr, buf, Unsafe.ARRAY_BYTE_BASE_OFFSET, keylen);
- return new String(buf, 0, keylen, StandardCharsets.UTF_8);
- }
-
- Node(long a, int kl, int h, int v, long[] b) {
- stat = new Stat(v);
- addr = a;
- keylen = kl;
- hash = h;
- System.arraycopy(b, 0, buf, 0, Math.ceilDiv(kl, 8));
- }
-
- boolean contentEquals(final long[] other_buf) {
- int k = keylen / 8;
- int r = keylen % 8;
- // Since the city name is most likely shorter than 16 characters
- // this should be faster than typical conditional checks
- long sum = 0;
- for (int i = 0; i < k; i++) {
- sum += buf[i] ^ other_buf[i];
- }
- sum += (buf[k] ^ other_buf[k]) & HASH_MASKS[r];
- return sum == 0;
- }
- }
-
- // split into chunks
- static long[] slice(long start_addr, long end_addr, long chunk_size, int cpu_cnt) {
- long[] ptrs = new long[cpu_cnt + 1];
- ptrs[0] = start_addr;
- for (int i = 1; i < cpu_cnt; i++) {
- long addr = start_addr + i * chunk_size;
- while (addr < end_addr && UNSAFE.getByte(addr++) != '\n')
- ;
- ptrs[i] = Math.min(addr, end_addr);
- }
- ptrs[cpu_cnt] = end_addr;
- return ptrs;
- }
-
- public static void main(String[] args) throws InterruptedException, IOException {
- int cpu_cnt = Runtime.getRuntime().availableProcessors() / 2;
- try (var file = FileChannel.open(Path.of(FILE), StandardOpenOption.READ)) {
- long start_addr = file.map(MapMode.READ_ONLY, 0, file.size(), Arena.global()).address();
- long file_size = file.size();
- long end_addr = start_addr + file_size;
- long chunk_size = Math.ceilDiv(file_size, cpu_cnt);
-
- // processing
- var threads = new Thread[cpu_cnt];
- var maps = new Node[cpu_cnt][];
- var ptrs = slice(start_addr, end_addr, chunk_size, cpu_cnt);
-
- for (int i = 0; i < cpu_cnt; i++) {
- int thread_id = i;
- long start = ptrs[i];
- long end = ptrs[i + 1];
- maps[i] = new Node[BUCKET_SIZE + 16]; // extra space for collisions
-
- (threads[i] = new Thread(() -> {
- long addr = start;
- var map = maps[thread_id];
- long[] buf = new long[13];
- // parse loop
- while (addr < end) {
- int idx = 0;
- long hash = 0;
- long word = 0;
- long row_addr = addr;
- int semi_pos = 8;
- while (semi_pos == 8) {
- word = UNSAFE.getLong(addr);
- buf[idx++] = word;
- // idea from thomaswue & royvanrijn
- long xor_semi = word ^ 0x3b3b3b3b3b3b3b3bL; // xor with ;;;;;;;;
- long semipos_code = (xor_semi - 0x0101010101010101L) & ~xor_semi & 0x8080808080808080L;
- semi_pos = Long.numberOfTrailingZeros(semipos_code) >>> 3;
- addr += semi_pos;
- hash ^= word & HASH_MASKS[semi_pos];
- }
-
- int hash32 = (int) (hash ^ (hash >>> 31));
- int keylen = (int) (addr - row_addr);
-
- // great idea from merykitty (Quan Anh Mai)
- long num_word = UNSAFE.getLong(++addr);
- int dot_pos = Long.numberOfTrailingZeros(~num_word & 0x10101000);
- addr += (dot_pos >>> 3) + 3;
-
- int shift = 28 - dot_pos;
- long signed = (~num_word << 59) >> 63;
- long dsmask = ~(signed & 0xFF);
- long digits = ((num_word & dsmask) << shift) & 0x0F000F0F00L;
- long abs_val = ((digits * 0x640a0001) >>> 32) & 0x3FF;
- int val = (int) ((abs_val ^ signed) - signed);
-
- int bucket = (hash32 & BUCKET_MASK);
- while (true) {
- var node = map[bucket];
- if (node == null) {
- map[bucket] = new Node(row_addr, keylen, hash32, val, buf);
- break;
- }
- if (node.keylen == keylen && node.hash == hash32 && node.contentEquals(buf)) {
- node.stat.add(val);
- break;
- }
- bucket++;
- }
- }
- })).start();
- }
-
- // join all
- for (var thread : threads)
- thread.join();
-
- // collect results
- TreeMap<String, Stat> ms = new TreeMap<>();
- for (var map : maps) {
- for (var node : map) {
- if (node == null)
- continue;
- var stat = ms.putIfAbsent(node.key(), node.stat);
- if (stat != null)
- stat.merge(node.stat);
- }
- }
-
- System.out.println(ms);
- }
- }
- } \ No newline at end of file
+package dev.morling.onebrc;
+
+import java.io.IOException;
+import java.lang.foreign.Arena;
+import java.lang.reflect.Field;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.TreeMap;
+import sun.misc.Unsafe;
+
+public class CalculateAverage_abeobk {
+ private static final String FILE = "./measurements.txt";
+ private static final int BUCKET_SIZE = 1 << 16;
+ private static final int BUCKET_MASK = BUCKET_SIZE - 1;
+ private static final int MAX_STR_LEN = 100;
+ private static final Unsafe UNSAFE = initUnsafe();
+ private static final long[] HASH_MASKS = new long[]{
+ 0x0L,
+ 0xffL,
+ 0xffffL,
+ 0xffffffL,
+ 0xffffffffL,
+ 0xffffffffffL,
+ 0xffffffffffffL,
+ 0xffffffffffffffL,
+ 0xffffffffffffffffL, };
+
+ private static Unsafe initUnsafe() {
+ try {
+ Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
+ theUnsafe.setAccessible(true);
+ return (Unsafe) theUnsafe.get(Unsafe.class);
+ }
+ catch (Exception ex) {
+ throw new RuntimeException();
+ }
+ }
+
+ // stat
+ private static class Stat {
+ private int min;
+ private int max;
+ private long sum;
+ private int count;
+
+ Stat(int v) {
+ sum = min = max = v;
+ count = 1;
+ }
+
+ void add(int val) {
+ min = Math.min(val, min);
+ max = Math.max(val, max);
+ sum += val;
+ count++;
+ }
+
+ void merge(Stat other) {
+ min = Math.min(other.min, min);
+ max = Math.max(other.max, max);
+ sum += other.sum;
+ count += other.count;
+ }
+
+ public String toString() {
+ return String.format("%.1f/%.1f/%.1f", min * 0.1, sum * 0.1 / count, max * 0.1);
+ }
+ }
+
+ static class Node {
+ long addr;
+ int keylen;
+ int hash;
+ long[] buf = new long[13];
+ Stat stat;
+
+ String key() {
+ byte[] buf = new byte[MAX_STR_LEN];
+ UNSAFE.copyMemory(null, addr, buf, Unsafe.ARRAY_BYTE_BASE_OFFSET, keylen);
+ return new String(buf, 0, keylen, StandardCharsets.UTF_8);
+ }
+
+ Node(long a, int kl, int h, int v, long[] b) {
+ stat = new Stat(v);
+ addr = a;
+ keylen = kl;
+ hash = h;
+ System.arraycopy(b, 0, buf, 0, Math.ceilDiv(kl, 8));
+ }
+
+ boolean contentEquals(final long[] other_buf) {
+ int k = keylen / 8;
+ int r = keylen % 8;
+ // Since the city name is most likely shorter than 16 characters
+ // this should be faster than typical conditional checks
+ long sum = 0;
+ for (int i = 0; i < k; i++) {
+ sum += buf[i] ^ other_buf[i];
+ }
+ sum += (buf[k] ^ other_buf[k]) & HASH_MASKS[r];
+ return sum == 0;
+ }
+ }
+
+ // split into chunks
+ static long[] slice(long start_addr, long end_addr, long chunk_size, int cpu_cnt) {
+ long[] ptrs = new long[cpu_cnt + 1];
+ ptrs[0] = start_addr;
+ for (int i = 1; i < cpu_cnt; i++) {
+ long addr = start_addr + i * chunk_size;
+ while (addr < end_addr && UNSAFE.getByte(addr++) != '\n')
+ ;
+ ptrs[i] = Math.min(addr, end_addr);
+ }
+ ptrs[cpu_cnt] = end_addr;
+ return ptrs;
+ }
+
+ public static void main(String[] args) throws InterruptedException, IOException {
+ int cpu_cnt = Runtime.getRuntime().availableProcessors() / 2;
+ try (var file = FileChannel.open(Path.of(FILE), StandardOpenOption.READ)) {
+ long start_addr = file.map(MapMode.READ_ONLY, 0, file.size(), Arena.global()).address();
+ long file_size = file.size();
+ long end_addr = start_addr + file_size;
+ long chunk_size = Math.ceilDiv(file_size, cpu_cnt);
+
+ // processing
+ var threads = new Thread[cpu_cnt];
+ var maps = new Node[cpu_cnt][];
+ var ptrs = slice(start_addr, end_addr, chunk_size, cpu_cnt);
+
+ for (int i = 0; i < cpu_cnt; i++) {
+ int thread_id = i;
+ long start = ptrs[i];
+ long end = ptrs[i + 1];
+ maps[i] = new Node[BUCKET_SIZE + 16]; // extra space for collisions
+
+ (threads[i] = new Thread(() -> {
+ long addr = start;
+ var map = maps[thread_id];
+ long[] buf = new long[13];
+ // parse loop
+ while (addr < end) {
+ int idx = 0;
+ long hash = 0;
+ long word = 0;
+ long row_addr = addr;
+ int semi_pos = 8;
+ while (semi_pos == 8) {
+ word = UNSAFE.getLong(addr);
+ buf[idx++] = word;
+ // idea from thomaswue & royvanrijn
+ long xor_semi = word ^ 0x3b3b3b3b3b3b3b3bL; // xor with ;;;;;;;;
+ long semipos_code = (xor_semi - 0x0101010101010101L) & ~xor_semi & 0x8080808080808080L;
+ semi_pos = Long.numberOfTrailingZeros(semipos_code) >>> 3;
+ addr += semi_pos;
+ hash ^= word & HASH_MASKS[semi_pos];
+ }
+
+ int hash32 = (int) (hash ^ (hash >>> 31));
+ int keylen = (int) (addr - row_addr);
+
+ // great idea from merykitty (Quan Anh Mai)
+ long num_word = UNSAFE.getLong(++addr);
+ int dot_pos = Long.numberOfTrailingZeros(~num_word & 0x10101000);
+ addr += (dot_pos >>> 3) + 3;
+
+ int shift = 28 - dot_pos;
+ long signed = (~num_word << 59) >> 63;
+ long dsmask = ~(signed & 0xFF);
+ long digits = ((num_word & dsmask) << shift) & 0x0F000F0F00L;
+ long abs_val = ((digits * 0x640a0001) >>> 32) & 0x3FF;
+ int val = (int) ((abs_val ^ signed) - signed);
+
+ int bucket = (hash32 & BUCKET_MASK);
+ while (true) {
+ var node = map[bucket];
+ if (node == null) {
+ map[bucket] = new Node(row_addr, keylen, hash32, val, buf);
+ break;
+ }
+ if (node.keylen == keylen && node.hash == hash32 && node.contentEquals(buf)) {
+ node.stat.add(val);
+ break;
+ }
+ bucket++;
+ }
+ }
+ })).start();
+ }
+
+ // join all
+ for (var thread : threads)
+ thread.join();
+
+ // collect results
+ TreeMap<String, Stat> ms = new TreeMap<>();
+ for (var map : maps) {
+ for (var node : map) {
+ if (node == null)
+ continue;
+ var stat = ms.putIfAbsent(node.key(), node.stat);
+ if (stat != null)
+ stat.merge(node.stat);
+ }
+ }
+
+ System.out.println(ms);
+ }
+ }
+} \ No newline at end of file