Graham McIntire
b4b8d4ec47
- Create MaidenheadChangesetHelpers: consolidate grid validation, callsign normalization, lat/lon validation, grid/latlon derivation across 6 schemas - Create ContextHelpers: shared fetch_owned with admin bypass, safe_enqueue for Oban workers, UUID casting to replace CastError rescues - Extend LiveHelpers: add current_user/1 (removes 7 duplicate definitions), subscribe/2 (replaces 13 inline PubSub sites), assign_url_params/2 - Extract Propagation.ScoreStore (528 lines): separate file I/O and cache management from scoring logic, 13 defdelegate passthroughs - Split SubmitLive (942->475 lines): extract CSV/ADIF upload rendering into 3 function component modules (csv_upload, adif_upload, preview) - Update 16 LiveViews to use shared helpers
528 lines
19 KiB
Elixir
528 lines
19 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoreStore do
|
||
@moduledoc false
|
||
|
||
alias Microwaveprop.Instrument
|
||
alias Microwaveprop.Propagation.Grid
|
||
alias Microwaveprop.Propagation.ProfilesFile
|
||
alias Microwaveprop.Propagation.ScoreCache
|
||
alias Microwaveprop.Propagation.ScoresFile
|
||
alias Microwaveprop.Weather.ScalarFile
|
||
|
||
require Logger
|
||
|
||
@hrrr_forecast_horizon_hours 48
|
||
@fallback_profile_lookback_hours 24
|
||
|
||
@doc """
|
||
Replace every propagation score for `valid_time` with `scores`.
|
||
|
||
Used by `PropagationGridWorker` on the hot path. Scores are written
|
||
as binary files on disk via `ScoresFile.write!/3`, one file per
|
||
band.
|
||
|
||
Consumes `scores` in a single streaming pass that folds each score
|
||
straight into a per-band accumulator. Previously this function ran
|
||
`Enum.to_list/1` followed by `Enum.group_by/2`, which held two full
|
||
copies of the ~460k-entry grid (list + grouped list) in memory at
|
||
once — the hot path's largest transient spike after native-duct
|
||
merge. The single-pass reduce keeps only one copy and buys back
|
||
~100 MB of headroom per forecast-hour step.
|
||
"""
|
||
@spec replace_scores(Enumerable.t(), DateTime.t()) :: {:ok, non_neg_integer()} | {:error, term()}
|
||
def replace_scores(scores, %DateTime{} = valid_time) do
|
||
do_replace_scores(scores, valid_time)
|
||
end
|
||
|
||
defp do_replace_scores(scores, valid_time) do
|
||
# Pure grouping phase runs outside the telemetry span — typically
|
||
# <10ms on small result sets, and the span's two dispatches
|
||
# (~100µs each) would otherwise dominate. The span now wraps only
|
||
# the per-band writes, which is where the actual DB cost lives.
|
||
{per_band, total} =
|
||
Enum.reduce(scores, {%{}, 0}, fn score, {acc, count} ->
|
||
{Map.update(acc, score.band_mhz, [score], &[score | &1]), count + 1}
|
||
end)
|
||
|
||
Instrument.span(
|
||
[:db, :replace_scores],
|
||
%{valid_time: valid_time},
|
||
fn ->
|
||
Enum.each(per_band, fn {band_mhz, band_scores} ->
|
||
try do
|
||
ScoresFile.write!(band_mhz, valid_time, band_scores)
|
||
rescue
|
||
e ->
|
||
Logger.warning("Propagation: ScoresFile write failed for band=#{band_mhz} vt=#{valid_time}: #{inspect(e)}")
|
||
end
|
||
end)
|
||
|
||
{:ok, total}
|
||
end
|
||
)
|
||
end
|
||
|
||
@doc """
|
||
Remove score files with valid_times older than 3 hours. Called on
|
||
a cron by `Microwaveprop.Workers.PropagationPruneWorker`.
|
||
|
||
The cutoff sits one hour beyond HRRR's ~2h publish lag: the hourly
|
||
seeder picks `run_time = now - 2h`, so the f00 analysis file is
|
||
written at valid_time = now - 2h. A 2h cutoff deletes it within
|
||
minutes; a 3h cutoff keeps it alive until the next hourly run
|
||
supersedes it.
|
||
"""
|
||
@spec prune_old_scores() :: :ok
|
||
def prune_old_scores do
|
||
cutoff = DateTime.shift(DateTime.utc_now(), hour: -3)
|
||
file_deleted = ScoresFile.prune_older_than(cutoff)
|
||
profiles_deleted = ProfilesFile.prune_older_than(cutoff)
|
||
scalar_deleted = ScalarFile.prune_older_than(cutoff)
|
||
|
||
total = file_deleted + profiles_deleted + scalar_deleted
|
||
|
||
if total > 0 do
|
||
Logger.info(
|
||
"PropagationScores: pruned #{file_deleted} old score files + " <>
|
||
"#{profiles_deleted} profile files + #{scalar_deleted} scalar dirs " <>
|
||
"(before #{cutoff})"
|
||
)
|
||
end
|
||
|
||
# Sweep orphaned .tmp.* files left by crashed atomic-write processes
|
||
tmp_deleted =
|
||
Enum.reduce([ScoresFile.base_dir(), ProfilesFile.base_dir(), ScalarFile.base_dir()], 0, &sweep_tmp_dir/2)
|
||
|
||
if tmp_deleted > 0 do
|
||
Logger.info("PropagationScores: swept #{tmp_deleted} orphaned .tmp files")
|
||
end
|
||
|
||
:ok
|
||
end
|
||
|
||
defp sweep_tmp_dir(dir, acc) do
|
||
case File.ls(dir) do
|
||
{:ok, entries} ->
|
||
Enum.reduce(entries, acc, &sweep_tmp_entry(dir, &1, &2))
|
||
|
||
_ ->
|
||
acc
|
||
end
|
||
end
|
||
|
||
defp sweep_tmp_entry(parent, entry, acc) do
|
||
full = Path.join(parent, entry)
|
||
|
||
case File.ls(full) do
|
||
{:ok, _} ->
|
||
sweep_tmp_dir(full, acc)
|
||
|
||
{:error, _} ->
|
||
if String.contains?(entry, ".tmp.") do
|
||
_ = File.rm_rf(full)
|
||
acc + 1
|
||
else
|
||
acc
|
||
end
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Retains score files through GEFS's 168-hour horizon and profile/scalar
|
||
files through HRRR's 48-hour horizon, deleting files older than
|
||
`run_time`. Called
|
||
by `NotifyListener` after chain completion to keep `/data/scores`
|
||
within bounds.
|
||
|
||
Mirrors `ScoreCache.prune_outside_window` for the on-disk tier.
|
||
"""
|
||
@spec retain_scores_window(DateTime.t()) :: :ok
|
||
def retain_scores_window(%DateTime{} = run_time) do
|
||
scores_deleted = ScoresFile.retain_window(run_time, 168)
|
||
profiles_deleted = ProfilesFile.retain_window(run_time, 48)
|
||
scalars_deleted = ScalarFile.retain_window(run_time, 48)
|
||
total = scores_deleted + profiles_deleted + scalars_deleted
|
||
|
||
if total > 0 do
|
||
Logger.info(
|
||
"Propagation.retain_scores_window: deleted #{total} stale files (#{scores_deleted} scores, #{profiles_deleted} profiles, #{scalars_deleted} scalars)"
|
||
)
|
||
end
|
||
|
||
:ok
|
||
end
|
||
|
||
@doc """
|
||
Returns distinct valid_times for a band, ordered ascending. Always
|
||
reads from the on-disk `ScoresFile` store — the `ScoreCache` only
|
||
holds whatever hours have been fetched or broadcast, which can be a
|
||
partial view of what's actually on disk, so using it as the source
|
||
of truth for the timeline makes new forecast hours invisible until
|
||
the cache happens to catch up. Filters out times more than 1 hour in
|
||
the past, but always includes the most recent valid_time so there's
|
||
always data to display.
|
||
"""
|
||
# HRRR forecast horizon: f00..f48 covers the next 48 hours from cycle
|
||
# time (f01-f18 hourly, f21-f48 3-hourly). Anything beyond that in the
|
||
# score store is a leftover from a stale cycle and clutters the
|
||
# timeline without adding information.
|
||
@spec available_valid_times(non_neg_integer()) :: [DateTime.t()]
|
||
def available_valid_times(band_mhz) do
|
||
{past_cutoff, future_cutoff} = hot_cache_window()
|
||
|
||
case ScoresFile.list_valid_times(band_mhz) do
|
||
[] -> []
|
||
times -> filter_or_latest(times, past_cutoff, future_cutoff)
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
The active forecast window for the `/map` UI: one hour in the past
|
||
through HRRR's 48-hour forecast horizon. Used by `NotifyListener` to
|
||
bound ETS growth — long-horizon GEFS `.prop` files on disk must not
|
||
balloon `propagation_score_cache` past the memory the UI actually
|
||
reads.
|
||
"""
|
||
@spec hot_cache_window() :: {DateTime.t(), DateTime.t()}
|
||
def hot_cache_window do
|
||
now = DateTime.utc_now()
|
||
past = DateTime.shift(now, hour: -1)
|
||
future = DateTime.shift(now, hour: @hrrr_forecast_horizon_hours)
|
||
{past, future}
|
||
end
|
||
|
||
defp filter_or_latest(times, past_cutoff, future_cutoff) do
|
||
fresh =
|
||
Enum.filter(times, fn t ->
|
||
DateTime.compare(t, past_cutoff) != :lt and DateTime.compare(t, future_cutoff) != :gt
|
||
end)
|
||
|
||
if fresh == [] do
|
||
[Enum.max(times, DateTime)]
|
||
else
|
||
fresh
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Get scores for a band at a specific valid_time, optionally within a bounding box.
|
||
If valid_time is nil, uses the earliest available (current analysis hour).
|
||
Excludes factors for performance.
|
||
"""
|
||
@spec scores_at(non_neg_integer(), DateTime.t() | nil, %{optional(String.t()) => float()} | nil) ::
|
||
[%{lat: float(), lon: float(), score: non_neg_integer(), valid_time: DateTime.t()}]
|
||
def scores_at(band_mhz, valid_time, bounds \\ nil) do
|
||
time = valid_time || earliest_valid_time(band_mhz)
|
||
|
||
case time do
|
||
nil -> []
|
||
_ -> scores_at_fetch(band_mhz, time, bounds)
|
||
end
|
||
end
|
||
|
||
# Cache-hit path is the map's most frequent LiveView call (~every
|
||
# pan + click). A wrapping Instrument.span fires 2 telemetry handler
|
||
# dispatches that dominate the ~10µs ETS lookup — skip the span on
|
||
# hits and rely on the cheap hit/miss counter for the cache-ratio
|
||
# panel. The miss path still wraps the disk read where duration is
|
||
# the meaningful signal.
|
||
defp scores_at_fetch(band_mhz, time, bounds) do
|
||
case ScoreCache.fetch_bounds(band_mhz, time, bounds) do
|
||
{:ok, scores} ->
|
||
:telemetry.execute([:microwaveprop, :propagation, :scores_at, :cache], %{}, %{hit: true})
|
||
Enum.map(scores, &Map.put(&1, :valid_time, time))
|
||
|
||
:miss ->
|
||
:telemetry.execute([:microwaveprop, :propagation, :scores_at, :cache], %{}, %{hit: false})
|
||
|
||
Instrument.span([:propagation, :scores_at], %{band_mhz: band_mhz}, fn ->
|
||
read_from_disk_and_cache(band_mhz, time, bounds)
|
||
end)
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Variant of `scores_at/3` that always reads from the `.prop` file on
|
||
disk and overwrites the cache entry, rather than returning whatever
|
||
the cache happens to hold. Use from update paths (the map's
|
||
`propagation_updated` handler) where the underlying file has just
|
||
been rewritten but the cache may still contain the previous chain's
|
||
scores because of the race between `propagation:cache` fan-out and
|
||
`propagation:updated` delivery.
|
||
"""
|
||
@spec scores_at_fresh(non_neg_integer(), DateTime.t(), %{optional(String.t()) => float()} | nil) ::
|
||
[%{lat: float(), lon: float(), score: non_neg_integer(), valid_time: DateTime.t()}]
|
||
def scores_at_fresh(band_mhz, %DateTime{} = valid_time, bounds \\ nil) do
|
||
read_from_disk_and_cache(band_mhz, valid_time, bounds)
|
||
end
|
||
|
||
defp read_from_disk_and_cache(band_mhz, time, bounds) do
|
||
full = ScoresFile.read_bounds(band_mhz, time)
|
||
ScoreCache.put(band_mhz, time, full)
|
||
|
||
full
|
||
|> filter_bounds(bounds)
|
||
|> Enum.map(&Map.put(&1, :valid_time, time))
|
||
end
|
||
|
||
@doc """
|
||
Load the full North America score set for `{band_mhz, valid_time}` from
|
||
the on-disk binary files (HRRR `.prop` + HRDPS `.hrdps.prop`, merged) and
|
||
broadcast it to every `ScoreCache` in the cluster. Called from
|
||
`PropagationGridWorker` after each forecast hour so all pods have a warm
|
||
cache by the time clients begin requesting the new hour.
|
||
|
||
Returns `{:error, :enoent}` only when neither file exists. A single
|
||
missing file (HRDPS pre-cycle, HRRR briefly absent) is OK — the cache
|
||
warms with whichever side is available.
|
||
"""
|
||
@spec warm_cache_and_broadcast(non_neg_integer(), DateTime.t()) ::
|
||
:ok | {:error, :enoent | :invalid_format}
|
||
def warm_cache_and_broadcast(band_mhz, valid_time) do
|
||
hrrr = read_score_points(&ScoresFile.read/2, band_mhz, valid_time)
|
||
hrdps = read_score_points(&ScoresFile.read_hrdps/2, band_mhz, valid_time)
|
||
|
||
# Merge priority: HRRR > HRDPS > GEFS. GEFS provides extended-horizon
|
||
# coverage beyond HRRR's 48h window. Within the f24-f48 overlap, cells
|
||
# already present in HRRR/HRDPS are skipped so the coarser GEFS scores
|
||
# don't override the higher-resolution ones.
|
||
merged = (hrrr || []) ++ (hrdps || [])
|
||
gefs = read_score_points(&ScoresFile.read_gefs/2, band_mhz, valid_time) || []
|
||
scores = ScoresFile.merge_preferred(merged, gefs)
|
||
|
||
case {hrrr, hrdps, scores} do
|
||
{nil, nil, []} ->
|
||
{:error, :enoent}
|
||
|
||
_ ->
|
||
ScoreCache.broadcast_put(band_mhz, valid_time, scores)
|
||
:ok
|
||
end
|
||
end
|
||
|
||
defp read_score_points(reader, band_mhz, valid_time) do
|
||
case reader.(band_mhz, valid_time) do
|
||
{:ok, payload} -> ScoresFile.extract_points(payload, nil)
|
||
_ -> nil
|
||
end
|
||
end
|
||
|
||
defp filter_bounds(scores, nil), do: scores
|
||
|
||
defp filter_bounds(scores, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
||
Enum.filter(scores, fn %{lat: lat, lon: lon} ->
|
||
lat >= s and lat <= n and lon >= w and lon <= e
|
||
end)
|
||
end
|
||
|
||
@doc "Get the latest scores for a band (alias for scores_at with earliest valid_time)."
|
||
@spec latest_scores(non_neg_integer(), %{optional(String.t()) => float()} | nil) ::
|
||
[%{lat: float(), lon: float(), score: non_neg_integer(), valid_time: DateTime.t()}]
|
||
def latest_scores(band_mhz, bounds \\ nil) do
|
||
scores_at(band_mhz, nil, bounds)
|
||
end
|
||
|
||
defp earliest_valid_time(band_mhz) do
|
||
case ScoresFile.list_valid_times(band_mhz) do
|
||
[earliest | _] -> earliest
|
||
[] -> nil
|
||
end
|
||
end
|
||
|
||
@doc "Get scores across all forecast hours for a single grid point (for sparkline)."
|
||
@spec point_forecast(non_neg_integer(), float(), float()) ::
|
||
[%{valid_time: DateTime.t(), score: non_neg_integer()}]
|
||
def point_forecast(band_mhz, lat, lon) do
|
||
Instrument.span([:propagation, :point_forecast], %{band_mhz: band_mhz}, fn ->
|
||
{snapped_lat, snapped_lon} = snap_to_grid(lat, lon)
|
||
now = DateTime.utc_now()
|
||
|
||
# Use the on-disk .prop file list as the authoritative timeline so
|
||
# the chart never falls behind the main-map timeline (which also
|
||
# reads the disk). The cache is still consulted per-hour for a
|
||
# fast score lookup; a miss falls through to the file.
|
||
# Fan the per-hour disk lookups across 4 tasks. Each ScoresFile
|
||
# read_point is an NFS stat + pread of ~100 bytes (keyed byte at
|
||
# row*cols+col), so the ceiling is NFS RTT × number of hours —
|
||
# sequential ran ~4–5× the wall time of the slowest read.
|
||
band_mhz
|
||
|> ScoresFile.list_valid_times()
|
||
|> forecast_window(now)
|
||
|> Task.async_stream(
|
||
&point_forecast_entry(band_mhz, &1, snapped_lat, snapped_lon),
|
||
max_concurrency: 4,
|
||
ordered: true,
|
||
timeout: 5_000
|
||
)
|
||
|> Enum.flat_map(fn
|
||
{:ok, nil} ->
|
||
[]
|
||
|
||
{:ok, entry} ->
|
||
[entry]
|
||
|
||
{:exit, reason} ->
|
||
Logger.error(
|
||
"Propagation.point_forecast async lookup failed: band_mhz=#{band_mhz} lat=#{snapped_lat} lon=#{snapped_lon} reason=#{inspect(reason)}"
|
||
)
|
||
|
||
[]
|
||
end)
|
||
end)
|
||
end
|
||
|
||
defp point_forecast_entry(band_mhz, valid_time, lat, lon) do
|
||
case ScoreCache.fetch_point(band_mhz, valid_time, lat, lon) do
|
||
{:ok, score} ->
|
||
%{valid_time: valid_time, score: score}
|
||
|
||
:miss ->
|
||
case ScoresFile.read_point(band_mhz, valid_time, lat, lon) do
|
||
nil -> nil
|
||
score -> %{valid_time: valid_time, score: score}
|
||
end
|
||
end
|
||
end
|
||
|
||
# Select the set of valid_times the forecast chart should render.
|
||
# Mirrors `available_valid_times`: keep everything from one hour
|
||
# before now onward so the most recent analysis hour (typically
|
||
# ~30–60 min behind wall clock due to HRRR publishing lag) sits at
|
||
# the left edge of the chart as "now". When every hour on disk is
|
||
# older than that cutoff, fall back to just the newest entry so the
|
||
# chart can still render a single data point.
|
||
defp forecast_window([], _now), do: []
|
||
|
||
defp forecast_window(times, now) do
|
||
past_cutoff = DateTime.shift(now, hour: -1)
|
||
future_cutoff = DateTime.shift(now, hour: @hrrr_forecast_horizon_hours)
|
||
filter_or_latest(times, past_cutoff, future_cutoff)
|
||
end
|
||
|
||
defp snap_to_grid(lat, lon) do
|
||
step = Grid.step()
|
||
{Float.round(Float.round(lat / step) * step, 3), Float.round(Float.round(lon / step) * step, 3)}
|
||
end
|
||
|
||
@doc """
|
||
Get the full score and factors for a specific grid point, snapped to
|
||
the nearest grid cell.
|
||
|
||
`:profile_source` describes where the factor breakdown came from:
|
||
|
||
* `:exact` — rescored from this `valid_time`'s own profile file.
|
||
* `{:fallback, fallback_valid_time}` — the requested hour's profile
|
||
was missing, so we rescored from the most recent analysis profile
|
||
within the lookback window. Treat as approximate.
|
||
* `:unavailable` — no profile available; `factors` is `%{}`.
|
||
"""
|
||
@spec point_detail(non_neg_integer(), float(), float(), DateTime.t() | nil) ::
|
||
%{
|
||
lat: float(),
|
||
lon: float(),
|
||
score: non_neg_integer(),
|
||
factors: map(),
|
||
profile_source: :exact | {:fallback, DateTime.t()} | :unavailable,
|
||
valid_time: DateTime.t()
|
||
}
|
||
| nil
|
||
def point_detail(band_mhz, lat, lon, valid_time \\ nil) do
|
||
Instrument.span([:propagation, :point_detail], %{band_mhz: band_mhz}, fn ->
|
||
do_point_detail(band_mhz, lat, lon, valid_time)
|
||
end)
|
||
end
|
||
|
||
defp do_point_detail(band_mhz, lat, lon, valid_time) do
|
||
{snapped_lat, snapped_lon} = snap_to_grid(lat, lon)
|
||
time = valid_time || latest_valid_time(band_mhz)
|
||
|
||
case time do
|
||
nil ->
|
||
nil
|
||
|
||
_ ->
|
||
case ScoresFile.read_point(band_mhz, time, snapped_lat, snapped_lon) do
|
||
nil ->
|
||
nil
|
||
|
||
score ->
|
||
{factors, source} = factors_for(band_mhz, time, snapped_lat, snapped_lon)
|
||
|
||
%{
|
||
lat: snapped_lat,
|
||
lon: snapped_lon,
|
||
score: score,
|
||
factors: factors,
|
||
profile_source: source,
|
||
valid_time: time
|
||
}
|
||
end
|
||
end
|
||
end
|
||
|
||
# Rebuild the factor breakdown for a clicked grid cell by rescoring
|
||
# the persisted HRRR profile.
|
||
#
|
||
# The Rust pipeline (`prop_grid_rs`) writes a per-cell profile file
|
||
# for every chain step (f00..f18 since Phase 2 cutover), so the
|
||
# `:exact` branch covers a healthy production state. The fallback to
|
||
# a recent analysis profile remains as a safety net for missed
|
||
# cycles or partial chain runs — when it kicks in we tag the result
|
||
# `{:fallback, profile_valid_time}` so the UI can label the
|
||
# breakdown as approximated rather than silently misrepresent it.
|
||
@spec factors_for(non_neg_integer(), DateTime.t(), float(), float()) ::
|
||
{map(), :exact | {:fallback, DateTime.t()} | :unavailable}
|
||
defp factors_for(band_mhz, valid_time, lat, lon) do
|
||
case ProfilesFile.read_point(valid_time, lat, lon) do
|
||
nil ->
|
||
factors_from_fallback_profile(band_mhz, valid_time, lat, lon)
|
||
|
||
profile ->
|
||
{Microwaveprop.Propagation.factors_from_profile(band_mhz, valid_time, profile, lat, lon), :exact}
|
||
end
|
||
end
|
||
|
||
defp factors_from_fallback_profile(band_mhz, valid_time, lat, lon) do
|
||
case latest_profile_time_within_lookback(valid_time) do
|
||
nil ->
|
||
{%{}, :unavailable}
|
||
|
||
fallback_time ->
|
||
case ProfilesFile.read_point(fallback_time, lat, lon) do
|
||
nil ->
|
||
{%{}, :unavailable}
|
||
|
||
profile ->
|
||
{Microwaveprop.Propagation.factors_from_profile(band_mhz, fallback_time, profile, lat, lon),
|
||
{:fallback, fallback_time}}
|
||
end
|
||
end
|
||
end
|
||
|
||
defp latest_profile_time_within_lookback(%DateTime{} = valid_time) do
|
||
lookback_cutoff = DateTime.shift(valid_time, hour: -@fallback_profile_lookback_hours)
|
||
|
||
ProfilesFile.list_valid_times()
|
||
|> Enum.filter(fn t ->
|
||
DateTime.compare(t, valid_time) != :gt and DateTime.compare(t, lookback_cutoff) != :lt
|
||
end)
|
||
|> case do
|
||
[] -> nil
|
||
past -> Enum.max(past, DateTime)
|
||
end
|
||
end
|
||
|
||
@doc "Get the latest valid_time across all bands."
|
||
@spec latest_valid_time() :: DateTime.t() | nil
|
||
def latest_valid_time do
|
||
ScoresFile.latest_valid_time()
|
||
end
|
||
|
||
@doc "Get the latest valid_time for a specific band."
|
||
@spec latest_valid_time(non_neg_integer()) :: DateTime.t() | nil
|
||
def latest_valid_time(band_mhz) do
|
||
case ScoresFile.list_valid_times(band_mhz) do
|
||
[] -> nil
|
||
times -> Enum.max(times, DateTime)
|
||
end
|
||
end
|
||
end
|