Word frequency in fifteen languages

const TEXT = `The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.`;

const words = TEXT.toLowerCase().match(/[a-z]+/g);
const counts = new Map();
for (const w of words) counts.set(w, (counts.get(w) ?? 0) + 1);

console.log(`words: ${words.length}, distinct: ${counts.size}`);
const top = [...counts]
  .sort(([a, m], [b, n]) => n - m || (a < b ? -1 : 1))
  .slice(0, 10);
for (const [word, n] of top) console.log(`${String(n).padStart(3)} ${word}`);

import re
from collections import Counter

TEXT = """The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind."""

words = re.findall(r"[a-z]+", TEXT.lower())
counts = Counter(words)

print(f"words: {len(words)}, distinct: {len(counts)}")
for word, n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:10]:
    print(f"{n:3d} {word}")

TEXT = <<~END
  The river rose in the night, and by morning the road to the mill was under
  water. The miller walked down to the edge of the water and looked at the mill:
  the wheel was turning faster than it had ever turned, and the river was loud.
  He thought of the grain in the loft, dry for now, and of the long road round by
  the bridge. A boat would be faster than the road, but the boat was at the mill.
  So he sat on a stone by the water, and waited for the river to make up its mind.
END

words = TEXT.downcase.scan(/[a-z]+/)
counts = words.tally

puts "words: #{words.size}, distinct: #{counts.size}"
counts.sort_by { |word, n| [-n, word] }.first(10).each do |word, n|
  puts format("%3d %s", n, word)
end

(def text "The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.")

(let [words (re-seq #"[a-z]+" (clojure.string/lower-case text))
      counts (frequencies words)]
  (println (str "words: " (count words) ", distinct: " (count counts)))
  (doseq [[word n] (->> counts
                        (sort-by (juxt (comp - val) key))
                        (take 10))]
    (println (str (.padStart (str n) 3) " " word))))

val TEXT = """The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind."""

fun main() {
    val words = Regex("[a-z]+").findAll(TEXT.lowercase()).map { it.value }.toList()
    val counts = words.groupingBy { it }.eachCount()

    println("words: ${words.size}, distinct: ${counts.size}")
    counts.entries
        .sortedWith(compareByDescending<Map.Entry<String, Int>> { it.value }.thenBy { it.key })
        .take(10)
        .forEach { (word, n) -> println("%3d %s".format(n, word)) }
}

text <- "The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind."

lower <- tolower(text)
words <- regmatches(lower, gregexpr("[a-z]+", lower))[[1]]
counts <- table(words)

cat(sprintf("words: %d, distinct: %d\n", length(words), length(counts)))
top <- head(counts[order(-counts, names(counts), method = "radix")], 10)
cat(sprintf("%3d %s\n", as.vector(top), names(top)), sep = "")

use strict;
use warnings;

my $text = <<'END';
The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.
END

my @words = lc($text) =~ /[a-z]+/g;
my %count;
$count{$_}++ for @words;

printf "words: %d, distinct: %d\n", scalar @words, scalar keys %count;
my @top = (sort { $count{$b} <=> $count{$a} || $a cmp $b } keys %count)[0 .. 9];
printf "%3d %s\n", $count{$_}, $_ for @top;

<?php
$text = <<<'END'
The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.
END;

preg_match_all('/[a-z]+/', strtolower($text), $m);
$words = $m[0];
$counts = array_count_values($words);

printf("words: %d, distinct: %d\n", count($words), count($counts));
uksort($counts, fn($a, $b) => $counts[$b] <=> $counts[$a] ?: strcmp($a, $b));
foreach (array_slice($counts, 0, 10, true) as $word => $n) {
    printf("%3d %s\n", $n, $word);
}

local text = [[
The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.
]]

local counts, total = {}, 0
for word in text:lower():gmatch("%a+") do
  counts[word] = (counts[word] or 0) + 1
  total = total + 1
end

local list = {}
for word, n in pairs(counts) do list[#list + 1] = { word = word, n = n } end
table.sort(list, function(a, b)
  if a.n ~= b.n then return a.n > b.n end
  return a.word < b.word
end)

print(("words: %d, distinct: %d"):format(total, #list))
for i = 1, 10 do print(("%3d %s"):format(list[i].n, list[i].word)) end

let text = {|The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.|}

let words s =
  let buf = Buffer.create 16 and acc = ref [] in
  let flush () = if Buffer.length buf > 0 then (acc := Buffer.contents buf :: !acc; Buffer.clear buf) in
  String.iter (fun c -> if c >= 'a' && c <= 'z' then Buffer.add_char buf c else flush ()) (String.lowercase_ascii s);
  flush ();
  List.rev !acc

let () =
  let ws = words text in
  let counts = Hashtbl.create 64 in
  List.iter (fun w -> Hashtbl.replace counts w (1 + Option.value ~default:0 (Hashtbl.find_opt counts w))) ws;
  Printf.printf "words: %d, distinct: %d\n" (List.length ws) (Hashtbl.length counts);
  Hashtbl.to_seq counts
  |> List.of_seq
  |> List.sort (fun (w1, n1) (w2, n2) -> if n1 <> n2 then compare n2 n1 else compare w1 w2)
  |> List.filteri (fun i _ -> i < 10)
  |> List.iter (fun (w, n) -> Printf.printf "%3d %s\n" n w)

#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

static const char *TEXT =
    "The river rose in the night, and by morning the road to the mill was under\n"
    "water. The miller walked down to the edge of the water and looked at the mill:\n"
    "the wheel was turning faster than it had ever turned, and the river was loud.\n"
    "He thought of the grain in the loft, dry for now, and of the long road round by\n"
    "the bridge. A boat would be faster than the road, but the boat was at the mill.\n"
    "So he sat on a stone by the water, and waited for the river to make up its mind.\n";

/* A fixed-size hash table with open addressing: plenty for one paragraph. */
#define SLOTS 1024
typedef struct { char word[32]; int n; } Entry;
static Entry table[SLOTS];
static int distinct;

static unsigned hash(const char *s) {  /* FNV-1a */
    unsigned h = 2166136261u;
    for (; *s; s++) h = (h ^ (unsigned char)*s) * 16777619u;
    return h;
}

static void count(const char *word) {
    unsigned i = hash(word) % SLOTS;
    while (table[i].n && strcmp(table[i].word, word) != 0) i = (i + 1) % SLOTS;
    if (!table[i].n) { strcpy(table[i].word, word); distinct++; }
    table[i].n++;
}

static int by_count_then_word(const void *pa, const void *pb) {
    const Entry *a = pa, *b = pb;
    if (a->n != b->n) return b->n - a->n;
    return strcmp(a->word, b->word);
}

int main(void) {
    char word[32];
    int len = 0, total = 0;
    for (const char *p = TEXT;; p++) {
        if (isalpha((unsigned char)*p) && len < 31) {
            word[len++] = tolower((unsigned char)*p);
        } else if (len) {
            word[len] = '\0';
            count(word);
            total++;
            len = 0;
        }
        if (!*p) break;
    }

    Entry *list = malloc(distinct * sizeof *list);
    int k = 0;
    for (int i = 0; i < SLOTS; i++)
        if (table[i].n) list[k++] = table[i];
    qsort(list, distinct, sizeof *list, by_count_then_word);

    printf("words: %d, distinct: %d\n", total, distinct);
    for (int i = 0; i < 10 && i < distinct; i++) printf("%3d %s\n", list[i].n, list[i].word);
    free(list);
    return 0;
}

use std::collections::HashMap;

const TEXT: &str = "The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.";

fn main() {
    let lower = TEXT.to_lowercase();
    let words: Vec<&str> = lower
        .split(|c: char| !c.is_ascii_lowercase())
        .filter(|w| !w.is_empty())
        .collect();

    let mut counts: HashMap<&str, usize> = HashMap::new();
    for w in &words {
        *counts.entry(w).or_insert(0) += 1;
    }

    println!("words: {}, distinct: {}", words.len(), counts.len());
    let mut top: Vec<(&str, usize)> = counts.into_iter().collect();
    top.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
    for (word, n) in top.iter().take(10) {
        println!("{n:3} {word}");
    }
}

package main

import (
	"cmp"
	"fmt"
	"slices"
	"strings"
)

const text = `The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.`

func main() {
	words := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool { return r < 'a' || r > 'z' })
	counts := map[string]int{}
	for _, w := range words {
		counts[w]++
	}

	fmt.Printf("words: %d, distinct: %d\n", len(words), len(counts))
	keys := make([]string, 0, len(counts))
	for w := range counts {
		keys = append(keys, w)
	}
	slices.SortFunc(keys, func(a, b string) int {
		return cmp.Or(cmp.Compare(counts[b], counts[a]), strings.Compare(a, b))
	})
	for _, w := range keys[:10] {
		fmt.Printf("%3d %s\n", counts[w], w)
	}
}

DROP TABLE IF EXISTS words;
CREATE TABLE words AS
WITH RECURSIVE
  doc(body) AS (SELECT 'The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.'),
  cleaned(s) AS (
    SELECT replace(replace(replace(replace(lower(body), char(10), ' '), ',', ' '), '.', ' '), ':', ' ') || ' '
    FROM doc
  ),
  split(word, rest) AS (
    SELECT '', s FROM cleaned
    UNION ALL
    SELECT substr(rest, 1, instr(rest, ' ') - 1), substr(rest, instr(rest, ' ') + 1)
    FROM split WHERE rest <> ''
  )
SELECT word FROM split WHERE word <> '';

SELECT 'words: ' || count(*) || ', distinct: ' || count(DISTINCT word) AS summary FROM words;

SELECT printf('%3d %s', count(*), word) AS top
FROM words
GROUP BY word
ORDER BY count(*) DESC, word
LIMIT 10;

WITH
  words AS (
    SELECT unnest(regexp_extract_all(lower('The river rose in the night, and by morning the road to the mill was under
water. The miller walked down to the edge of the water and looked at the mill:
the wheel was turning faster than it had ever turned, and the river was loud.
He thought of the grain in the loft, dry for now, and of the long road round by
the bridge. A boat would be faster than the road, but the boat was at the mill.
So he sat on a stone by the water, and waited for the river to make up its mind.'), '[a-z]+')) AS word
  ),
  counts AS (SELECT word, count(*) AS n FROM words GROUP BY word),
  lines AS (
    SELECT 0 AS rank, 'words: ' || (SELECT count(*) FROM words) || ', distinct: ' || count(*) AS line
    FROM counts
    UNION ALL
    SELECT row_number() OVER (ORDER BY n DESC, word), printf('%3d %s', n, word)
    FROM counts
  )
SELECT line FROM lines WHERE rank <= 10 ORDER BY rank;