Chunk batch inserts to avoid Postgres parameter limit, add ANALYZE migration
This commit is contained in:
parent
b4012baf89
commit
33490d37d0
2 changed files with 28 additions and 12 deletions
|
|
@ -211,19 +211,26 @@ defmodule Microwaveprop.Weather do
|
||||||
def upsert_hrrr_profiles_batch(profiles) do
|
def upsert_hrrr_profiles_batch(profiles) do
|
||||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||||
|
|
||||||
entries =
|
profiles
|
||||||
Enum.map(profiles, fn attrs ->
|
|> Enum.chunk_every(500)
|
||||||
Map.merge(attrs, %{
|
|> Enum.reduce({0, nil}, fn chunk, {total_count, _} ->
|
||||||
id: Ecto.UUID.generate(),
|
entries =
|
||||||
inserted_at: now,
|
Enum.map(chunk, fn attrs ->
|
||||||
updated_at: now
|
Map.merge(attrs, %{
|
||||||
})
|
id: Ecto.UUID.generate(),
|
||||||
end)
|
inserted_at: now,
|
||||||
|
updated_at: now
|
||||||
|
})
|
||||||
|
end)
|
||||||
|
|
||||||
Repo.insert_all(HrrrProfile, entries,
|
{count, rows} =
|
||||||
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
Repo.insert_all(HrrrProfile, entries,
|
||||||
conflict_target: [:lat, :lon, :valid_time]
|
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
||||||
)
|
conflict_target: [:lat, :lon, :valid_time]
|
||||||
|
)
|
||||||
|
|
||||||
|
{total_count + count, rows}
|
||||||
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
def has_hrrr_profile?(lat, lon, valid_time) do
|
def has_hrrr_profile?(lat, lon, valid_time) do
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
defmodule Microwaveprop.Repo.Migrations.AnalyzeAllTables do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def up do
|
||||||
|
execute "ANALYZE"
|
||||||
|
end
|
||||||
|
|
||||||
|
def down, do: :ok
|
||||||
|
end
|
||||||
Loading…
Add table
Reference in a new issue