650 lines
19 KiB
Elixir
650 lines
19 KiB
Elixir
defmodule ToweropsWeb.GraphLive.ShowEventsTest do
|
|
@moduledoc """
|
|
Additional coverage for `ToweropsWeb.GraphLive.Show` focusing on:
|
|
* check-based graph paths (`check_id`)
|
|
* range switching across all valid ranges
|
|
* handle_info handlers (live_poll, sensor/neighbor/status events)
|
|
* terminate cleanup
|
|
* less-common sensor types (voltage, storage volume, count, wireless, unknown)
|
|
* access control / mismatched check_device
|
|
"""
|
|
use ToweropsWeb.ConnCase
|
|
|
|
import Phoenix.LiveViewTest
|
|
|
|
alias Towerops.Monitoring
|
|
alias Towerops.Monitoring.Check
|
|
alias Towerops.Monitoring.CheckResult
|
|
alias Towerops.Organizations
|
|
alias Towerops.Repo
|
|
alias Towerops.Sites
|
|
alias Towerops.Snmp
|
|
alias Towerops.Snmp.Device, as: SnmpDevice
|
|
alias Towerops.Snmp.Interface
|
|
alias Towerops.Snmp.Sensor
|
|
alias Towerops.Snmp.Storage
|
|
alias Towerops.Snmp.StorageReading
|
|
|
|
setup :register_and_log_in_user
|
|
|
|
setup %{user: user} do
|
|
{:ok, organization} = Organizations.create_organization(%{name: "Events Org"}, user.id)
|
|
|
|
{:ok, site} =
|
|
Sites.create_site(%{name: "Events Site", organization_id: organization.id})
|
|
|
|
{:ok, device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Events Router",
|
|
ip_address: "192.168.50.1",
|
|
site_id: site.id,
|
|
organization_id: organization.id,
|
|
snmp_enabled: true
|
|
})
|
|
|
|
snmp_device =
|
|
%SnmpDevice{}
|
|
|> SnmpDevice.changeset(%{
|
|
device_id: device.id,
|
|
sys_name: "events-router",
|
|
sys_descr: "Events Router",
|
|
sys_object_id: "1.3.6.1.4.1.9"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
%{organization: organization, site: site, device: device, snmp_device: snmp_device}
|
|
end
|
|
|
|
defp default_config_for("http"), do: %{"url" => "https://example.com"}
|
|
defp default_config_for("tcp"), do: %{"host" => "example.com", "port" => 80}
|
|
defp default_config_for("dns"), do: %{"hostname" => "example.com"}
|
|
defp default_config_for("ssl"), do: %{"host" => "example.com"}
|
|
defp default_config_for("ping"), do: %{"host" => "example.com"}
|
|
defp default_config_for(_), do: %{}
|
|
|
|
defp insert_check!(device, attrs) do
|
|
check_type = Map.get(attrs, :check_type, "http")
|
|
config = Map.get(attrs, :config, default_config_for(check_type))
|
|
|
|
base = %{
|
|
device_id: device.id,
|
|
organization_id: device.organization_id,
|
|
name: "Test Check",
|
|
check_type: check_type,
|
|
enabled: true,
|
|
interval_seconds: 60,
|
|
config: config
|
|
}
|
|
|
|
%Check{}
|
|
|> Check.changeset(Map.merge(base, attrs))
|
|
|> Repo.insert!()
|
|
end
|
|
|
|
defp insert_check_result!(check, attrs) do
|
|
base = %{
|
|
check_id: check.id,
|
|
organization_id: check.organization_id,
|
|
checked_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
status: 0,
|
|
value: 12.5
|
|
}
|
|
|
|
%CheckResult{}
|
|
|> CheckResult.changeset(Map.merge(base, attrs))
|
|
|> Repo.insert!()
|
|
end
|
|
|
|
describe "check-based graph (check_id route)" do
|
|
test "renders http check graph with results", %{conn: conn, device: device} do
|
|
check = insert_check!(device, %{name: "HTTP up", check_type: "http"})
|
|
insert_check_result!(check, %{value: 25.4})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
assert html =~ "HTTP up"
|
|
assert html =~ device.name
|
|
end
|
|
|
|
test "renders dns check graph (dual axis with status)", %{conn: conn, device: device} do
|
|
check = insert_check!(device, %{name: "DNS lookup", check_type: "dns"})
|
|
insert_check_result!(check, %{status: 0, value: 8.2})
|
|
insert_check_result!(check, %{status: 2, value: nil})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
assert html =~ "DNS lookup"
|
|
end
|
|
|
|
test "renders dns check graph with no results (empty path)", %{conn: conn, device: device} do
|
|
check = insert_check!(device, %{name: "Empty DNS", check_type: "dns"})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
assert html =~ "Empty DNS"
|
|
end
|
|
|
|
test "renders snmp_processor check graph (no auto scale)", %{conn: conn, device: device} do
|
|
check = insert_check!(device, %{name: "CPU check", check_type: "snmp_processor"})
|
|
insert_check_result!(check, %{value: 55.0})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
assert html =~ "CPU check"
|
|
end
|
|
|
|
test "renders snmp_sensor check using sensor_unit from config", %{conn: conn, device: device} do
|
|
check =
|
|
insert_check!(device, %{
|
|
name: "Sensor C",
|
|
check_type: "snmp_sensor",
|
|
config: %{"sensor_unit" => "Celsius"}
|
|
})
|
|
|
|
insert_check_result!(check, %{value: 30.0})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
assert html =~ "Sensor C"
|
|
end
|
|
|
|
test "redirects when check is not found", %{conn: conn, device: device} do
|
|
fake_check_id = Ecto.UUID.generate()
|
|
|
|
assert {:error, {:live_redirect, %{to: target}}} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{fake_check_id}")
|
|
|
|
assert target =~ "/devices/#{device.id}"
|
|
end
|
|
|
|
test "redirects when check belongs to different device", %{
|
|
conn: conn,
|
|
device: device,
|
|
organization: organization,
|
|
site: site
|
|
} do
|
|
{:ok, other_device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Other",
|
|
ip_address: "10.0.0.2",
|
|
site_id: site.id,
|
|
organization_id: organization.id,
|
|
snmp_enabled: true
|
|
})
|
|
|
|
check = insert_check!(other_device, %{name: "Other Check"})
|
|
|
|
assert {:error, {:live_redirect, %{to: target}}} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
assert target =~ "/devices/#{device.id}"
|
|
end
|
|
|
|
test "honors range param on check graph", %{conn: conn, device: device} do
|
|
check = insert_check!(device, %{name: "HTTP w/ range", check_type: "http"})
|
|
insert_check_result!(check, %{value: 10.0})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}&range=7d")
|
|
|
|
assert html =~ "7 Days"
|
|
assert html =~ "HTTP w/ range"
|
|
end
|
|
|
|
test "change_range on check graph patches with check_id preserved", %{
|
|
conn: conn,
|
|
device: device
|
|
} do
|
|
check = insert_check!(device, %{name: "Range Patch", check_type: "http"})
|
|
|
|
{:ok, view, _html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/check?check_id=#{check.id}")
|
|
|
|
html =
|
|
view
|
|
|> element("button", "1 Hour")
|
|
|> render_click()
|
|
|
|
assert html =~ "1 Hour"
|
|
assert html =~ "Range Patch"
|
|
end
|
|
end
|
|
|
|
describe "Range switching" do
|
|
test "supports all valid ranges via URL parameter", %{conn: conn, device: device} do
|
|
cases = [
|
|
{"1h", "1 Hour"},
|
|
{"6h", "6 Hours"},
|
|
{"12h", "12 Hours"},
|
|
{"24h", "24 Hours"},
|
|
{"7d", "7 Days"},
|
|
{"30d", "30 Days"}
|
|
]
|
|
|
|
for {range, label} <- cases do
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/latency?range=#{range}")
|
|
|
|
assert html =~ label, "expected range=#{range} to render '#{label}'"
|
|
end
|
|
end
|
|
|
|
test "unknown range falls back gracefully (defaults to 24h behavior)", %{
|
|
conn: conn,
|
|
device: device
|
|
} do
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/latency?range=bogus")
|
|
|
|
# Page renders without crashing
|
|
assert html =~ device.name
|
|
end
|
|
|
|
test "change_range click for 7d updates label", %{conn: conn, device: device} do
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/latency")
|
|
|
|
html =
|
|
view
|
|
|> element("button", "7 Days")
|
|
|> render_click()
|
|
|
|
assert html =~ "7 Days"
|
|
end
|
|
|
|
test "change_range preserves sensor_id when present", %{
|
|
conn: conn,
|
|
device: device,
|
|
snmp_device: snmp_device
|
|
} do
|
|
sensor =
|
|
%Sensor{}
|
|
|> Sensor.changeset(%{
|
|
snmp_device_id: snmp_device.id,
|
|
sensor_type: "voltage",
|
|
sensor_index: "1",
|
|
sensor_oid: "1.3.6.1.4.1.9.9.13.1.2.1.3.1",
|
|
sensor_descr: "PSU1 V",
|
|
sensor_unit: "V"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
Snmp.create_sensor_reading(%{
|
|
sensor_id: sensor.id,
|
|
value: 12.0,
|
|
checked_at: DateTime.truncate(DateTime.utc_now(), :second)
|
|
})
|
|
|
|
{:ok, view, _html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/voltage?sensor_id=#{sensor.id}")
|
|
|
|
html =
|
|
view
|
|
|> element("button", "1 Hour")
|
|
|> render_click()
|
|
|
|
assert html =~ "1 Hour"
|
|
assert html =~ "PSU1 V"
|
|
end
|
|
|
|
test "change_range preserves storage_id when present", %{
|
|
conn: conn,
|
|
device: device,
|
|
snmp_device: snmp_device
|
|
} do
|
|
storage =
|
|
%Storage{}
|
|
|> Storage.changeset(%{
|
|
snmp_device_id: snmp_device.id,
|
|
storage_index: 1,
|
|
description: "/",
|
|
storage_type: "fixed_disk",
|
|
total_bytes: 1_000_000,
|
|
used_bytes: 500_000
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
%StorageReading{}
|
|
|> StorageReading.changeset(%{
|
|
storage_id: storage.id,
|
|
used_bytes: 500_000,
|
|
total_bytes: 1_000_000,
|
|
usage_percent: 50.0,
|
|
checked_at: DateTime.truncate(DateTime.utc_now(), :second)
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
{:ok, view, _html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/storage_volume?storage_id=#{storage.id}")
|
|
|
|
html =
|
|
view
|
|
|> element("button", "6 Hours")
|
|
|> render_click()
|
|
|
|
assert html =~ "6 Hours"
|
|
end
|
|
end
|
|
|
|
describe "Sensor type rendering" do
|
|
test "renders voltage chart config", %{conn: conn, device: device} do
|
|
{:ok, _view, html} = live(conn, ~p"/devices/#{device.id}/graph/voltage")
|
|
assert html =~ "Voltage"
|
|
end
|
|
|
|
test "renders count chart config", %{conn: conn, device: device} do
|
|
{:ok, _view, html} = live(conn, ~p"/devices/#{device.id}/graph/count")
|
|
assert html =~ "Count"
|
|
end
|
|
|
|
test "renders pppoe_sessions chart config", %{conn: conn, device: device} do
|
|
{:ok, _view, html} = live(conn, ~p"/devices/#{device.id}/graph/pppoe_sessions")
|
|
assert html =~ "PPPoE Sessions"
|
|
end
|
|
|
|
test "renders connections chart config", %{conn: conn, device: device} do
|
|
{:ok, _view, html} = live(conn, ~p"/devices/#{device.id}/graph/connections")
|
|
assert html =~ "Connections"
|
|
end
|
|
|
|
test "renders wireless chart config", %{conn: conn, device: device} do
|
|
{:ok, _view, html} = live(conn, ~p"/devices/#{device.id}/graph/wireless")
|
|
assert html =~ "Wireless Metrics"
|
|
end
|
|
|
|
test "humanizes unknown sensor_type", %{conn: conn, device: device} do
|
|
{:ok, _view, html} = live(conn, ~p"/devices/#{device.id}/graph/foo_bar_baz")
|
|
assert html =~ "Foo Bar Baz"
|
|
end
|
|
|
|
test "renders storage_volume graph for given storage", %{
|
|
conn: conn,
|
|
device: device,
|
|
snmp_device: snmp_device
|
|
} do
|
|
storage =
|
|
%Storage{}
|
|
|> Storage.changeset(%{
|
|
snmp_device_id: snmp_device.id,
|
|
storage_index: 2,
|
|
description: "/data",
|
|
storage_type: "fixed_disk",
|
|
total_bytes: 10_000,
|
|
used_bytes: 2_000
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
%StorageReading{}
|
|
|> StorageReading.changeset(%{
|
|
storage_id: storage.id,
|
|
used_bytes: 2_000,
|
|
total_bytes: 10_000,
|
|
usage_percent: 20.0,
|
|
checked_at: now
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/storage_volume?storage_id=#{storage.id}")
|
|
|
|
assert html =~ "Storage Usage"
|
|
assert html =~ "/data"
|
|
end
|
|
end
|
|
|
|
describe "handle_info / live polling" do
|
|
test "device_status_changed reloads device assign", %{conn: conn, device: device} do
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/latency")
|
|
|
|
send(view.pid, {:device_status_changed, device.id, :down, 9999})
|
|
|
|
# Process should still be alive
|
|
assert Process.alive?(view.pid)
|
|
assert render(view) =~ device.name
|
|
end
|
|
|
|
test "state_sensors_updated triggers reload without crashing", %{
|
|
conn: conn,
|
|
device: device,
|
|
snmp_device: snmp_device
|
|
} do
|
|
sensor =
|
|
%Sensor{}
|
|
|> Sensor.changeset(%{
|
|
snmp_device_id: snmp_device.id,
|
|
sensor_type: "cpu_load",
|
|
sensor_index: "1",
|
|
sensor_oid: "1.3.6.1.4.1.9.9.109.1.1.1.1.8.1",
|
|
sensor_descr: "CPU 0",
|
|
sensor_unit: "%"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
Snmp.create_sensor_reading(%{
|
|
sensor_id: sensor.id,
|
|
value: 30.0,
|
|
checked_at: DateTime.truncate(DateTime.utc_now(), :second)
|
|
})
|
|
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/processors")
|
|
|
|
send(view.pid, {:state_sensors_updated, device.id})
|
|
|
|
assert render(view) =~ "Processor Usage"
|
|
end
|
|
|
|
test "neighbors_updated is a no-op", %{conn: conn, device: device} do
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/latency")
|
|
|
|
send(view.pid, {:neighbors_updated, device.id})
|
|
assert render(view) =~ device.name
|
|
end
|
|
|
|
test "live_poll while not in live mode is ignored", %{conn: conn, device: device} do
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/latency")
|
|
|
|
send(view.pid, :live_poll)
|
|
assert render(view) =~ device.name
|
|
end
|
|
|
|
test "unrecognized info message is ignored (catch-all)", %{conn: conn, device: device} do
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/latency")
|
|
|
|
send(view.pid, :something_unexpected)
|
|
assert render(view) =~ device.name
|
|
end
|
|
|
|
test "terminate cancels any live polling timer", %{conn: conn, device: device} do
|
|
Process.flag(:trap_exit, true)
|
|
{:ok, view, _html} = live(conn, ~p"/devices/#{device.id}/graph/latency")
|
|
|
|
pid = view.pid
|
|
assert Process.alive?(pid)
|
|
|
|
# cancel_live_polling_timer/1 must safely handle the case where no timer
|
|
# was set (the latency graph isn't in live mode). terminate/2 calls it.
|
|
ref = Process.monitor(pid)
|
|
GenServer.stop(pid, :shutdown, 1_000)
|
|
|
|
assert_receive {:DOWN, ^ref, :process, _, _}, 1_000
|
|
end
|
|
end
|
|
|
|
describe "access control extras" do
|
|
test "redirects when device belongs to a different organization", %{
|
|
conn: conn,
|
|
user: user
|
|
} do
|
|
# Create a separate org/site/device the current user does not belong to
|
|
other_user = Towerops.AccountsFixtures.user_fixture()
|
|
|
|
{:ok, other_org} =
|
|
Organizations.create_organization(%{name: "Other Org"}, other_user.id)
|
|
|
|
{:ok, other_site} =
|
|
Sites.create_site(%{name: "Other Site", organization_id: other_org.id})
|
|
|
|
{:ok, foreign_device} =
|
|
Towerops.Devices.create_device(%{
|
|
name: "Forbidden",
|
|
ip_address: "10.20.30.40",
|
|
site_id: other_site.id,
|
|
organization_id: other_org.id,
|
|
snmp_enabled: true
|
|
})
|
|
|
|
_ = user
|
|
|
|
assert {:error, {:live_redirect, %{to: "/devices"}}} =
|
|
live(conn, ~p"/devices/#{foreign_device.id}/graph/latency")
|
|
end
|
|
|
|
test "redirects when sensor-route device id is not found", %{conn: conn} do
|
|
fake_id = Ecto.UUID.generate()
|
|
|
|
assert {:error, {:live_redirect, %{to: "/devices"}}} =
|
|
live(conn, ~p"/devices/#{fake_id}/graph/processors")
|
|
end
|
|
|
|
test "renders gracefully when sensor_id query param does not exist", %{
|
|
conn: conn,
|
|
device: device
|
|
} do
|
|
fake_sensor_id = Ecto.UUID.generate()
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/temperature?sensor_id=#{fake_sensor_id}")
|
|
|
|
assert is_binary(html) and byte_size(html) > 0
|
|
end
|
|
|
|
test "renders gracefully when storage_id query param does not exist", %{
|
|
conn: conn,
|
|
device: device
|
|
} do
|
|
fake_storage_id = Ecto.UUID.generate()
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/storage_volume?storage_id=#{fake_storage_id}")
|
|
|
|
assert is_binary(html) and byte_size(html) > 0
|
|
end
|
|
|
|
test "renders interface_errors with non-existent interface id", %{
|
|
conn: conn,
|
|
device: device
|
|
} do
|
|
fake_interface_id = Ecto.UUID.generate()
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/interface_errors?interface_id=#{fake_interface_id}")
|
|
|
|
assert is_binary(html) and byte_size(html) > 0
|
|
end
|
|
|
|
test "renders traffic with non-existent interface id", %{
|
|
conn: conn,
|
|
device: device
|
|
} do
|
|
fake_interface_id = Ecto.UUID.generate()
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/traffic?interface_id=#{fake_interface_id}")
|
|
|
|
assert is_binary(html) and byte_size(html) > 0
|
|
end
|
|
end
|
|
|
|
describe "interface_errors graph data shapes" do
|
|
setup %{snmp_device: snmp_device} do
|
|
interface =
|
|
%Interface{}
|
|
|> Interface.changeset(%{
|
|
snmp_device_id: snmp_device.id,
|
|
if_index: 2,
|
|
if_descr: "eth1",
|
|
if_name: "eth1",
|
|
if_oper_status: "up",
|
|
if_speed: 1_000_000_000
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
%{interface: interface}
|
|
end
|
|
|
|
test "computes error rate-of-change between two stats", %{
|
|
conn: conn,
|
|
device: device,
|
|
interface: interface
|
|
} do
|
|
base = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
Snmp.create_interface_stat(%{
|
|
interface_id: interface.id,
|
|
if_in_errors: 0,
|
|
if_out_errors: 0,
|
|
if_in_discards: 0,
|
|
if_out_discards: 0,
|
|
checked_at: DateTime.add(base, -60, :second)
|
|
})
|
|
|
|
Snmp.create_interface_stat(%{
|
|
interface_id: interface.id,
|
|
if_in_errors: 12,
|
|
if_out_errors: 6,
|
|
if_in_discards: 3,
|
|
if_out_discards: 0,
|
|
checked_at: base
|
|
})
|
|
|
|
{:ok, _view, html} =
|
|
live(conn, ~p"/devices/#{device.id}/graph/interface_errors?interface_id=#{interface.id}")
|
|
|
|
assert html =~ "Interface Errors & Discards"
|
|
end
|
|
end
|
|
|
|
describe "monitoring helper coverage" do
|
|
test "get_check_graph_data returns rows for snmp_sensor with source backfill path", %{
|
|
device: device,
|
|
snmp_device: snmp_device
|
|
} do
|
|
sensor =
|
|
%Sensor{}
|
|
|> Sensor.changeset(%{
|
|
snmp_device_id: snmp_device.id,
|
|
sensor_type: "temperature",
|
|
sensor_index: "1",
|
|
sensor_oid: "1.3.6.1.4.1.1",
|
|
sensor_descr: "Temp1",
|
|
sensor_unit: "C"
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
check =
|
|
insert_check!(device, %{
|
|
name: "SNMP backfill",
|
|
check_type: "snmp_sensor",
|
|
source_id: sensor.id,
|
|
config: %{"sensor_unit" => "Celsius"}
|
|
})
|
|
|
|
Snmp.create_sensor_reading(%{
|
|
sensor_id: sensor.id,
|
|
value: 22.5,
|
|
checked_at: DateTime.add(DateTime.utc_now(), -1, :hour)
|
|
})
|
|
|
|
from = DateTime.add(DateTime.utc_now(), -2, :hour)
|
|
to = DateTime.utc_now()
|
|
points = Monitoring.get_check_graph_data(check.id, from, to)
|
|
assert is_list(points)
|
|
end
|
|
end
|
|
end
|