Chunk batch inserts to avoid Postgres parameter limit, add ANALYZE migration
This commit is contained in:
parent
b4012baf89
commit
33490d37d0
2 changed files with 28 additions and 12 deletions
|
|
@ -211,19 +211,26 @@ defmodule Microwaveprop.Weather do
|
|||
def upsert_hrrr_profiles_batch(profiles) do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
entries =
|
||||
Enum.map(profiles, fn attrs ->
|
||||
Map.merge(attrs, %{
|
||||
id: Ecto.UUID.generate(),
|
||||
inserted_at: now,
|
||||
updated_at: now
|
||||
})
|
||||
end)
|
||||
profiles
|
||||
|> Enum.chunk_every(500)
|
||||
|> Enum.reduce({0, nil}, fn chunk, {total_count, _} ->
|
||||
entries =
|
||||
Enum.map(chunk, fn attrs ->
|
||||
Map.merge(attrs, %{
|
||||
id: Ecto.UUID.generate(),
|
||||
inserted_at: now,
|
||||
updated_at: now
|
||||
})
|
||||
end)
|
||||
|
||||
Repo.insert_all(HrrrProfile, entries,
|
||||
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
||||
conflict_target: [:lat, :lon, :valid_time]
|
||||
)
|
||||
{count, rows} =
|
||||
Repo.insert_all(HrrrProfile, entries,
|
||||
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
||||
conflict_target: [:lat, :lon, :valid_time]
|
||||
)
|
||||
|
||||
{total_count + count, rows}
|
||||
end)
|
||||
end
|
||||
|
||||
def has_hrrr_profile?(lat, lon, valid_time) do
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.AnalyzeAllTables do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
execute "ANALYZE"
|
||||
end
|
||||
|
||||
def down, do: :ok
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue