Add mix parse_file task, fix datetime deprecation, update gitignore, fix cleanup worker test
- Fix PacketCleanupWorkerTest to use relative timestamps based on configured retention period rather than hardcoded 365-day assumption (was failing when PACKET_RETENTION_DAYS env var is set to a small value) - Fix capture_aprs.py to use timezone-aware datetime.now(timezone.utc) instead of deprecated datetime.utcnow() - Add bad_packets.txt to .gitignore - Add mix task aprs.parse_file
This commit is contained in:
parent
790f13cef6
commit
9b1fa3ce50
4 changed files with 83 additions and 6 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -36,6 +36,7 @@ npm-debug.log
|
||||||
/.lexical
|
/.lexical
|
||||||
/badpackets.txt
|
/badpackets.txt
|
||||||
/packets.txt
|
/packets.txt
|
||||||
|
/bad_packets.txt
|
||||||
|
|
||||||
# Ignore Kubernetes secrets file
|
# Ignore Kubernetes secrets file
|
||||||
k8s/secrets.yaml
|
k8s/secrets.yaml
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ Uses receive-only login (passcode -1) so no amateur license required.
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
import signal
|
import signal
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
HOST = "dallas.aprs2.net"
|
HOST = "dallas.aprs2.net"
|
||||||
PORT = 10152 # Full feed, no filter required (use 14580 with a filter)
|
PORT = 10152 # Full feed, no filter required (use 14580 with a filter)
|
||||||
|
|
@ -52,7 +52,7 @@ def main():
|
||||||
count = 0
|
count = 0
|
||||||
with open(OUTPUT_FILE, "w", encoding="utf-8") as out:
|
with open(OUTPUT_FILE, "w", encoding="utf-8") as out:
|
||||||
out.write(f"# APRS capture from {HOST}:{PORT}\n")
|
out.write(f"# APRS capture from {HOST}:{PORT}\n")
|
||||||
out.write(f"# Started: {datetime.utcnow().isoformat()}Z\n\n")
|
out.write(f"# Started: {datetime.now(timezone.utc).isoformat()}\n\n")
|
||||||
|
|
||||||
while running:
|
while running:
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
71
lib/mix/tasks/aprs.parse_file.ex
Normal file
71
lib/mix/tasks/aprs.parse_file.ex
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
defmodule Mix.Tasks.Aprs.ParseFile do
|
||||||
|
@shortdoc "Parse APRS packets from a file and report failures"
|
||||||
|
|
||||||
|
@moduledoc """
|
||||||
|
Reads APRS packets from a text file, attempts to parse each one, and writes
|
||||||
|
any failures to an output file.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
mix aprs.parse_file [input_file] [--output output_file]
|
||||||
|
|
||||||
|
## Options
|
||||||
|
|
||||||
|
* `--output` / `-o` - path for the output file (default: `bad_packets.txt`)
|
||||||
|
|
||||||
|
## Arguments
|
||||||
|
|
||||||
|
* `input_file` - path to input file (default: `packets.txt`)
|
||||||
|
|
||||||
|
Lines starting with `#` or blank lines are skipped.
|
||||||
|
"""
|
||||||
|
|
||||||
|
use Mix.Task
|
||||||
|
|
||||||
|
@default_input "packets.txt"
|
||||||
|
@default_output "bad_packets.txt"
|
||||||
|
|
||||||
|
@impl Mix.Task
|
||||||
|
def run(args) do
|
||||||
|
{opts, positional, _} =
|
||||||
|
OptionParser.parse(args,
|
||||||
|
aliases: [o: :output],
|
||||||
|
strict: [output: :string]
|
||||||
|
)
|
||||||
|
|
||||||
|
input_file = List.first(positional) || @default_input
|
||||||
|
output_file = Keyword.get(opts, :output, @default_output)
|
||||||
|
|
||||||
|
if !File.exists?(input_file) do
|
||||||
|
Mix.shell().error("Error: input file not found: #{input_file}")
|
||||||
|
exit({:shutdown, 1})
|
||||||
|
end
|
||||||
|
|
||||||
|
Mix.Task.run("app.start")
|
||||||
|
|
||||||
|
{total, failures} =
|
||||||
|
input_file
|
||||||
|
|> File.stream!()
|
||||||
|
|> Stream.map(&String.trim/1)
|
||||||
|
|> Stream.reject(&skip_line?/1)
|
||||||
|
|> Enum.reduce({0, []}, fn line, {count, bad} ->
|
||||||
|
case Aprs.parse(line) do
|
||||||
|
{:ok, _} -> {count + 1, bad}
|
||||||
|
{:error, _} -> {count + 1, [line | bad]}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|
||||||
|
failures = Enum.reverse(failures)
|
||||||
|
pass_count = total - length(failures)
|
||||||
|
fail_count = length(failures)
|
||||||
|
|
||||||
|
File.write!(output_file, Enum.join(failures, "\n") <> if(failures == [], do: "", else: "\n"))
|
||||||
|
|
||||||
|
Mix.shell().info("Processed #{total} packets: #{pass_count} passed, #{fail_count} failed")
|
||||||
|
Mix.shell().info("Output written to: #{output_file}")
|
||||||
|
end
|
||||||
|
|
||||||
|
defp skip_line?(""), do: true
|
||||||
|
defp skip_line?("#" <> _), do: true
|
||||||
|
defp skip_line?(_), do: false
|
||||||
|
end
|
||||||
|
|
@ -45,14 +45,19 @@ defmodule Aprsme.Workers.PacketCleanupWorkerTest do
|
||||||
|
|
||||||
describe "perform/1 without cleanup_days" do
|
describe "perform/1 without cleanup_days" do
|
||||||
test "cleans up packets using the default retention period" do
|
test "cleans up packets using the default retention period" do
|
||||||
|
retention_days = Application.get_env(:aprsme, :packet_retention_days, 365)
|
||||||
job_args = %{}
|
job_args = %{}
|
||||||
|
|
||||||
# Create very old packets that should be deleted (older than 365 days)
|
# Create old packets that should be deleted (older than retention period)
|
||||||
old_time = DateTime.utc_now() |> DateTime.add(-400 * 86_400, :second) |> DateTime.truncate(:second)
|
old_time =
|
||||||
|
DateTime.utc_now()
|
||||||
|
|> DateTime.add(-(retention_days + 10) * 86_400, :second)
|
||||||
|
|> DateTime.truncate(:second)
|
||||||
|
|
||||||
insert_packets(10, %{received_at: old_time})
|
insert_packets(10, %{received_at: old_time})
|
||||||
|
|
||||||
# Create recent packets that should NOT be deleted
|
# Create recent packets that should NOT be deleted (1 hour ago)
|
||||||
recent_time = DateTime.utc_now() |> DateTime.add(-5 * 86_400, :second) |> DateTime.truncate(:second)
|
recent_time = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
|
||||||
insert_packets(3, %{received_at: recent_time})
|
insert_packets(3, %{received_at: recent_time})
|
||||||
|
|
||||||
assert :ok = PacketCleanupWorker.perform(job_args)
|
assert :ok = PacketCleanupWorker.perform(job_args)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue