Skip to content

Commit

Permalink
Starting work on broadway plugin (akoutmos#39)
Browse files Browse the repository at this point in the history
* Starting work on broadway plugin

* Added new init event

* Added additional datapoints

* Added additional init event metrics

* Added broadway test modules

* Move metrics for broadway

* Updated prom_ex module docs

* Updated Broadway events

* Example app test updates

* Added template for broadway dashboard

* Reworking Broadway plugin events

* Initial implementation of Broadway dashboard

* Fixing dashboard template

* Last touches on Broadway dashboard

* Wrapped up broadway tests
  • Loading branch information
akoutmos authored Dec 22, 2021
1 parent 729061f commit f4ebaa5
Show file tree
Hide file tree
Showing 21 changed files with 5,970 additions and 105 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added

- Updated BEAM plugin to surface JIT support
- Broadway metrics plugin
- Broadway Grafana dashboard

### Fixed

Expand Down
21 changes: 11 additions & 10 deletions example_applications/web_app/config/dev.exs
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,17 @@ config :web_app, WebAppWeb.Endpoint,
http: [port: 4000],
debug_errors: true,
code_reloader: true,
check_origin: false,
watchers: [
node: [
"node_modules/webpack/bin/webpack.js",
"--mode",
"development",
"--watch-stdin",
cd: Path.expand("../assets", __DIR__)
]
]
check_origin: false

# watchers: [
# node: [
# "node_modules/webpack/bin/webpack.js",
# "--mode",
# "development",
# "--watch-stdin",
# cd: Path.expand("../assets", __DIR__)
# ]
# ]

config :web_app, WebAppWeb.InternalEndpoint,
http: [port: 4001],
Expand Down
8 changes: 4 additions & 4 deletions example_applications/web_app/lib/web_app/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ defmodule WebApp.Application do

# Start the Endpoint (http/https)
WebAppWeb.Endpoint,
WebAppWeb.InternalEndpoint
WebAppWeb.InternalEndpoint,

# Work generator for Oban
# WebApp.RandomWorkGenerator

# Work generator for Broadway
# WebApp.TempTracker,
# WebApp.CityProducer,
# WebApp.TempProcessor
WebApp.TempTracker,
WebApp.CityProducer,
WebApp.TempProcessor
]

# See https://hexdocs.pm/elixir/Supervisor.html
Expand Down
247 changes: 247 additions & 0 deletions example_applications/web_app/lib/web_app/city_producer.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
defmodule WebApp.CityProducer do
@moduledoc false

use GenStage

require Logger

def start_link(_args) do
GenStage.start_link(__MODULE__, [], name: __MODULE__)
end

def init(_args) do
{:producer, city_list()}
end

def handle_demand(demand, state) when demand > length(state) do
handle_demand(demand, state ++ city_list())
end

def handle_demand(demand, state) do
{to_dispatch, remaining} = Enum.split(state, demand)

{:noreply, to_dispatch, remaining}
end

defp city_list do
[
{"Abu Dhabi", "United Arab Emirates"},
{"Abuja", "Nigeria"},
{"Accra", "Ghana"},
{"Adamstown", "Pitcairn Islands"},
{"Addis Ababa", "Ethiopia"},
{"Algiers", "Algeria"},
{"Alofi", "Niue"},
{"Amman", "Jordan"},
{"Amsterdam", "Netherlands"},
{"Andorra la Vella", "Andorra"},
{"Ankara", "Turkey"},
{"Antananarivo", "Madagascar"},
{"Apia", "Samoa"},
{"Ashgabat", "Turkmenistan"},
{"Asmara", "Eritrea"},
{"Asunción", "Paraguay"},
{"Athens", "Greece"},
{"Avarua", "Cook Islands"},
{"Baghdad", "Iraq"},
{"Baku", "Azerbaijan"},
{"Bamako", "Mali"},
{"Bandar Seri Begawan", "Brunei"},
{"Bangkok", "Thailand"},
{"Bangui", "Central African Republic"},
{"Banjul", "Gambia"},
{"Basseterre", "Saint Kitts and Nevis"},
{"Beijing", "China"},
{"Beirut", "Lebanon"},
{"Belgrade", "Serbia"},
{"Belmopan", "Belize"},
{"Berlin", "Germany"},
{"Bern", "Switzerland"},
{"Bishkek", "Kyrgyzstan"},
{"Bissau", "Guinea-Bissau"},
{"Bogotá", "Colombia"},
{"Brasília", "Brazil"},
{"Bratislava", "Slovakia"},
{"Brazzaville", "Republic of the Congo"},
{"Bridgetown", "Barbados"},
{"Brussels", "Belgium"},
{"Bucharest", "Romania"},
{"Budapest", "Hungary"},
{"Buenos Aires", "Argentina"},
{"Cairo", "Egypt"},
{"Canberra", "Australia"},
{"Caracas", "Venezuela"},
{"Castries", "Saint Lucia"},
{"Chișinău", "Moldova"},
{"Cockburn Town", "Turks and Caicos Islands"},
{"Conakry", "Guinea"},
{"Copenhagen", "Denmark"},
{"Dakar", "Senegal"},
{"Damascus", "Syria"},
{"Dhaka", "Bangladesh"},
{"Dili", "East Timor"},
{"Djibouti", "Djibouti"},
{"Dodoma", "Tanzania"},
{"Doha", "Qatar"},
{"Douglas", "Isle of Man"},
{"Dublin", "Ireland"},
{"Dushanbe", "Tajikistan"},
{"Flying Fish Cove", "Christmas Island"},
{"Freetown", "Sierra Leone"},
{"Funafuti", "Tuvalu"},
{"Gaborone", "Botswana"},
{"George Town", "Cayman Islands"},
{"Georgetown", "Ascension Island"},
{"Georgetown", "Guyana"},
{"Gibraltar", "Gibraltar"},
{"Gitega", "Burundi"},
{"Guatemala City", "Guatemala"},
{"Hagatna", "Guam"},
{"Hamilton", "Bermuda"},
{"Hanoi", "Vietnam"},
{"Harare", "Zimbabwe"},
{"Havana", "Cuba"},
{"Helsinki", "Finland"},
{"Honiara", "Solomon Islands"},
{"Islamabad", "Pakistan"},
{"Jakarta", "Indonesia"},
{"Jerusalem", "Israel"},
{"Juba", "South Sudan"},
{"Kabul", "Afghanistan"},
{"Kampala", "Uganda"},
{"Kathmandu", "Nepal"},
{"Khartoum", "Sudan"},
{"Kiev", "Ukraine"},
{"Kigali", "Rwanda"},
{"Kingston", "Jamaica"},
{"Kingston", "Norfolk Island"},
{"Kingstown", "Saint Vincent and the Grenadines"},
{"Kinshasa", "Democratic Republic of the Congo"},
{"Kuala Lumpur", "Malaysia"},
{"Kuwait City", "Kuwait"},
{"Libreville", "Gabon"},
{"Lilongwe", "Malawi"},
{"Lima", "Peru"},
{"Lisbon", "Portugal"},
{"Ljubljana", "Slovenia"},
{"Lomé", "Togo"},
{"London", "United Kingdom"},
{"Luanda", "Angola"},
{"Lusaka", "Zambia"},
{"Luxembourg", "Luxembourg"},
{"Madrid", "Spain"},
{"Majuro", "Marshall Islands"},
{"Malabo", "Equatorial Guinea"},
{"Malé", "Maldives"},
{"Managua", "Nicaragua"},
{"Manama", "Bahrain"},
{"Manila", "Philippines"},
{"Maputo", "Mozambique"},
{"Marigot", "Saint Martin"},
{"Maseru", "Lesotho"},
{"Mata-Utu", "Wallis and Futuna"},
{"Mbabane", "Eswatini"},
{"Mexico City", "Mexico"},
{"Minsk", "Belarus"},
{"Mogadishu", "Somalia"},
{"Monaco", "Monaco"},
{"Monrovia", "Liberia"},
{"Montevideo", "Uruguay"},
{"Moroni", "Comoros"},
{"Moscow", "Russia"},
{"Muscat", "Oman"},
{"Nairobi", "Kenya"},
{"Nassau", "Bahamas"},
{"Naypyidaw", "Myanmar"},
{"N'Djamena", "Chad"},
{"New Delhi", "India"},
{"Niamey", "Niger"},
{"Nouakchott", "Mauritania"},
{"Nouméa", "New Caledonia"},
{"Nuuk", "Greenland"},
{"Oranjestad", "Aruba"},
{"Oslo", "Norway"},
{"Ottawa", "Canada"},
{"Ouagadougou", "Burkina Faso"},
{"Pago Pago", "American Samoa"},
{"Panama City", "Panama"},
{"Papeete", "French Polynesia"},
{"Paramaribo", "Suriname"},
{"Paris", "France"},
{"Philipsburg", "Sint Maarten"},
{"Phnom Penh", "Cambodia"},
{"Plymouth", "Montserrat"},
{"Podgorica", "Montenegro"},
{"Port Louis", "Mauritius"},
{"Port Moresby", "Papua New Guinea"},
{"Port-au-Prince", "Haiti"},
{"Port of Spain", "Trinidad and Tobago"},
{"Porto-Novo", "Benin"},
{"Prague", "Czech Republic"},
{"Praia", "Cape Verde"},
{"Pretoria", "South Africa"},
{"Pyongyang", "North Korea"},
{"Quito", "Ecuador"},
{"Rabat", "Morocco"},
{"Ramallah", "Palestine"},
{"Reykjavík", "Iceland"},
{"Riga", "Latvia"},
{"Riyadh", "Saudi Arabia"},
{"Rome", "Italy"},
{"Roseau", "Dominica"},
{"Saipan", "Northern Mariana Islands"},
{"San José", "Costa Rica"},
{"San Juan", "Puerto Rico"},
{"San Marino", "San Marino"},
{"San Salvador", "El Salvador"},
{"Sana'a", "Yemen"},
{"Santiago", "Chile"},
{"Santo Domingo", "Dominican Republic"},
{"São Tomé", "São Tomé and Príncipe"},
{"Seoul", "South Korea"},
{"Singapore", "Singapore"},
{"Skopje", "North Macedonia"},
{"Sofia", "Bulgaria"},
{"St. George's", "Grenada"},
{"St. Pierre", "Saint Pierre and Miquelon"},
{"Stanley", "Falkland Islands"},
{"Stepanakert", "Artsakh"},
{"Stockholm", "Sweden"},
{"Sucre", "Bolivia"},
{"Suva", "Fiji"},
{"Taipei", "Taiwan"},
{"Tallinn", "Estonia"},
{"Tamar", "Hong Kong"},
{"Tarawa", "Kiribati"},
{"Tashkent", "Uzbekistan"},
{"Tbilisi", "Georgia"},
{"Tegucigalpa", "Honduras"},
{"Tehran", "Iran"},
{"Thimphu", "Bhutan"},
{"Tirana", "Albania"},
{"Tiraspol", "Transnistria"},
{"Tokyo", "Japan"},
{"Tripoli", "Libya"},
{"Tunis", "Tunisia"},
{"Ulaanbaatar", "Mongolia"},
{"Vaduz", "Liechtenstein"},
{"Valletta", "Malta"},
{"The Valley", "Anguilla"},
{"Victoria", "Seychelles"},
{"Vienna", "Austria"},
{"Vientiane", "Laos"},
{"Vilnius", "Lithuania"},
{"Warsaw", "Poland"},
{"Washington, D.C.", "United States"},
{"Wellington", "New Zealand"},
{"West Island", "Cocos (Keeling) Islands"},
{"Willemstad", "Curaçao"},
{"Windhoek", "Namibia"},
{"Yamoussoukro", "Ivory Coast"},
{"Yaoundé", "Cameroon"},
{"Yaren", "Nauru"},
{"Yerevan", "Armenia"},
{"Zagreb", "Croatia"}
]
end
end
46 changes: 24 additions & 22 deletions example_applications/web_app/lib/web_app/prom_ex.ex
Original file line number Diff line number Diff line change
Expand Up @@ -60,27 +60,27 @@ defmodule WebApp.PromEx do
def plugins do
[
# PromEx built in plugins
Plugins.Application,
Plugins.Beam,
{Plugins.Phoenix, endpoint: WebAppWeb.Endpoint, router: WebAppWeb.Router},
# OR
# {
# Plugins.Phoenix,
# endpoints: [
# {WebAppWeb.Endpoint,
# routers: [WebAppWeb.Router], additional_routes: @additional_routes},
# {WebAppWeb.InternalEndpoint,
# routers: [WebAppWeb.InternalRouter], event_prefix: [:internal, :endpoint]}
# ]
# },
{Plugins.Ecto, repos: [WebApp.Repo, WebApp.Repo2]},
{Plugins.Oban, oban_supervisors: [Oban, Oban.SuperSecret]},
Plugins.PhoenixLiveView,
{Plugins.PlugRouter, routers: [WebAppWeb.Router], event_prefix: [:phoenix, :endpoint]},
{Plugins.PlugCowboy,
ignore_routes: ["/metrics"], routers: [WebAppWeb.Router, WebAppWeb.InternalRouter]}

# PromEx.Plugins.Broadway
# Plugins.Application,
# Plugins.Beam,
# {Plugins.Phoenix, endpoint: WebAppWeb.Endpoint, router: WebAppWeb.Router},
# # OR
# # {
# # Plugins.Phoenix,
# # endpoints: [
# # {WebAppWeb.Endpoint,
# # routers: [WebAppWeb.Router], additional_routes: @additional_routes},
# # {WebAppWeb.InternalEndpoint,
# # routers: [WebAppWeb.InternalRouter], event_prefix: [:internal, :endpoint]}
# # ]
# # },
# {Plugins.Ecto, repos: [WebApp.Repo, WebApp.Repo2]},
# {Plugins.Oban, oban_supervisors: [Oban, Oban.SuperSecret]},
# Plugins.PhoenixLiveView,
# {Plugins.PlugRouter, routers: [WebAppWeb.Router], event_prefix: [:phoenix, :endpoint]},
# {Plugins.PlugCowboy,
# ignore_routes: ["/metrics"], routers: [WebAppWeb.Router, WebAppWeb.InternalRouter]}

PromEx.Plugins.Broadway

# Add your own PromEx metrics plugins
# WebApp.Users.PromEx
Expand All @@ -99,13 +99,15 @@ defmodule WebApp.PromEx do
[
# PromEx built in dashboard definitions. Remove dashboards that you do not need
{:prom_ex, "application.json"},
{:prom_ex, "absinthe.json"},
{:prom_ex, "beam.json"},
{:prom_ex, "phoenix.json"},
{:prom_ex, "ecto.json"},
{:prom_ex, "oban.json"},
{:prom_ex, "phoenix_live_view.json"},
{:prom_ex, "plug_cowboy.json"},
{:prom_ex, "plug_router.json"}
{:prom_ex, "plug_router.json"},
{:prom_ex, "broadway.json"}

# Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"}
# {:web_app, "/grafana_dashboards/user_metrics.json"}
Expand Down
Loading

0 comments on commit f4ebaa5

Please sign in to comment.