I'm trying to implement something like what is described in this answer, but I am getting errors like what I've included below when I compile the application.
** (Mix) Could not start application workers: Workers.Application.start(:normal, []) returned an error: shutdown: failed to start child: {Workers.UrlSupervisor, 2}
** (EXIT) already started: #PID<0.1034.0>
I'm not sure if I am inherently doing something I'm not allowed to here, or I've just made a little mistake.
For some context here are the supervisors:
defmodule Workers.Application do
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
#moduledoc false
use Application
def start(_type, _args) do
import Supervisor.Spec, warn: false
url_workers = 1..100 |> Enum.map(fn (i) -> supervisor(Workers.UrlSupervisor, [i], [id: {Workers.UrlSupervisor, i}, restart: :temporary]) end)
domain_workers = 1..100 |> Enum.map(fn (i) -> supervisor(Workers.DomainSupervisor, [i], [id: {Workers.DomainSupervisor, i}, restart: :temporary]) end)
opts = [strategy: :one_for_one, name: Workers.Supervisor]
Supervisor.start_link(url_workers ++ domain_workers, opts)
end
end
defmodule Workers.UrlSupervisor do
def start_link(id) do
import Supervisor.Spec, warn: false
children = [worker(Task, [&Workers.Url.worker/0], [id: {Workers.Url, id}, restart: :permanent])]
opts = [strategy: :one_for_one, name: Workers.UrlSupervisor]
Supervisor.start_link(children, opts)
end
end
defmodule Workers.DomainSupervisor do
def start_link(id) do
import Supervisor.Spec, warn: false
children = [worker(Task, [&Workers.Domain.worker/0], [id: {Workers.Domain, id}, restart: :permanent])]
opts = [strategy: :one_for_one, name: Workers.DomainSupervisor]
Supervisor.start_link(children, opts)
end
end
And here is one of the workers (they look largely the same).
defmodule Workers.Domain do
def worker do
case Store.Domains.pop do
:empty ->
IO.puts "[Domain] none found, waiting..."
:timer.sleep(1000)
{crawl_id, domain} ->
IO.puts "[Domains] found a domain to check: #{domain}"
case Core.check_domain(domain) do
:error ->
Utils.insert(crawl_id, domain, false)
:registered ->
Utils.insert(crawl_id, domain, false)
:available ->
Utils.insert(crawl_id, domain, true)
end
end
worker()
end
end
In your Workers.Application when starting Supervisors, you'r providing unique ids, but they also should have unique names.
Try adding another keyword, something like name: :"url_supervisor_#{i}":
def start(_type, _args) do
import Supervisor.Spec, warn: false
url_workers = 1..100 |> Enum.map(fn (i) ->
supervisor(Workers.UrlSupervisor, [i],
[id: {Workers.UrlSupervisor, i},
name: :"url_supervisor_#{i}", # Name added here
restart: :temporary])
end)
domain_workers = 1..100 |> Enum.map(fn (i) ->
supervisor(Workers.DomainSupervisor, [i],
[id: {Workers.DomainSupervisor, i},
name: :"domain_supervisor_#{i}", # Name added here
restart: :temporary])
end)
opts = [strategy: :one_for_one, name: Workers.Supervisor]
Supervisor.start_link(url_workers ++ domain_workers, opts)
end
Related
I'm an Erlang/Elixir noob and I'm doing a research on how to use the otel_resource_detector in Elixir.
I've managed to get this working in Erlang some time ago, but I'm struggling to get things right in Elixir.
This is what I have in Erlang:
-module(extra_metadata).
-behaviour(otel_resource_detector).
-export([get_resource/1]).
get_resource(_) ->
Resource1 = otel_resource:create(otel_resource_app_env:parse(get_metadata("/data/extrametadata.properties")), []),
{ok, HiddenMetadataFile} = file:read_file("/data/hiddenpath.properties"),
Resource2 = otel_resource:create(otel_resource_app_env:parse(get_metadata(HiddenMetadataFile)), []),
otel_resource:merge(Resource1, Resource2).
get_metadata(FileName) ->
try
{ok, MetadataFile} = file:read_file(FileName),
Lines = binary:split(MetadataFile, <<"\n">>, [trim, global]),
make_tuples(Lines, [])
catch _:_ -> "Extra Metadata not found"
end.
make_tuples([Line|Lines], Acc) ->
[Key, Value] = binary:split(Line, <<"=">>),
make_tuples(Lines, [{Key, Value}|Acc]);
make_tuples([], Acc) -> Acc.
Full Erlang app here: https://github.com/julianocosta89/erlang_otel_hello_server/tree/main
I'm trying to make use of the otel_resource_detector from OpenTelemetry:
https://github.com/open-telemetry/opentelemetry-erlang/blob/37f3cecd9ad2a7b8f3b94c89118585991f0023b1/apps/opentelemetry/src/otel_resource_detector.erl
How would I use that in Elixir?
One can easily call erlang modules from elixir. The first module would look like
defmodule ExtraMetadata do
#behaviour :otel_resource_detector
def get_resource(_) do
resource1 =
:otel_resource.create(
:otel_resource_app_env.parse(
get_metadata("/data/extrametadata.properties")), [])
{:ok, hidden_metadata_file} =
File.read("/data/hiddenpath.properties")
resource2 =
:otel_resource.create(
:otel_resource_app_env.parse(
get_metadata(hidden_metadata_file)), [])
:otel_resource.merge(resource1, resource2)
end
defp get_metadata(file_name) do
try do
{:ok, metadata_file} = File.read(file_name)
lines = :binary.split(metadata_file, <<"\n">>, [:trim, :global])
make_tuples(lines)
catch
_ -> "Extra Metadata not found"
end
end
# Enum.map/2 would be probably more idiomatic
defp make_tuples(lines, acc \\ [])
defp make_tuples([line|lines], acc) do
[key, value] = :binary.split(line, <<"=">>)
make_tuples(lines, [{key, value}|cc])
defp make_tuples([], acc), do: acc
end
NB I obviously did not check the code above, some glitches might need some additional handling.
Here is a working snippet:
defmodule ExtraMetadata do
#behaviour :otel_resource_detector
def get_resource(_) do
lines = read_file("/data/extrametadata.properties") |> unwrap_lines
file_path = read_file("/data/hiddenpath.properties") |> unwrap_lines
lines2 = read_file(file_path) |> unwrap_lines
attributes = get_attributes(Enum.concat(lines, lines2))
:otel_resource.create(attributes)
end
defp unwrap_lines({:ok, lines}), do: lines
defp unwrap_lines({:error, _}), do: []
defp read_file(file_name) do
try do
{:ok, String.split(File.read!(file_name), "\n")}
rescue
File.Error ->
{:error, "File does not exist, safe to continue"}
end
end
defp get_attributes(lines) do
# Transform each string into a tuple
Enum.map(lines, fn(line) ->
if String.length(line) > 0 do
[key, value] = String.split(line, "=")
{key, value}
else
{:error, "Empty string"}
end
end)
end
end
I've also pushed the whole project here: https://github.com/julianocosta89/elixir-hello.
I'm attempting to pass in an array to my Excel file Generator, however I recieve errors such as:
RuntimeError: Not an array ref in call to write_row()
when I attempt to run my script, can anyone tell me how best to pass in data to the following code?
module Accounting
module Datev
module Utils
class ExcelGenerator
require 'writeexcel'
# this class generates excel document and saves it with new path
# Example:
# d = [
# { name: 'Dima', dob: '27/02', city: 'Berlin' },
# { name: 'Conor', dob: '23/04', city: 'Berlin' },
# ]
# Accounting::Datev::Utils::ExcelGenerator.new(data: d, path: '~/text.xlsx').call
# will create excel file text.xlsx in the home directory with the following content
# | name | dob | city |
# | Dima | 27/02 | Berlin |
# | Conor | 23/04 | Berlin |
#
attr_reader :data, :path, :workbook, :worksheet
#array = [
{ name: 'Dima', dob: '27/02', city: 'Berlin' },
{ name: 'Conor', dob: '23/04', city: 'Berlin' }, ]
def initialize(data:, path:)
#data = data
#path = path
end
def call
# TODO: implement me``
create_new_workbook
add_worksheet
write_new_worksheet
save_file
end
def create_new_workbook
#workbook = WriteExcel.new('a_simple.xls')
end
def add_worksheet
#worksheet = workbook.add_worksheet
end
def write_new_worksheet
worksheet.write_row('A1', '#{array}')
end
def save_file
workbook.close
end
end
end
end
end
You didn't need to try convert array to string. But you should use different array format and use write_col method:
def initialize(data:, path:)
#data = data
#path = path
#array = [
['name', 'dob', 'city'],
['Dima', '27/02', 'Berlin'],
['Conor', '23/04','Berlin']
]
end
...
def write_new_worksheet
worksheet.write_col('A1', #array)
end
You can found more information here http://writeexcel.web.fc2.com/en/doc_en.html#write_col
Given I have the following array:
operations = [
[
:do_this,
["a"]
],
[
:do_that,
["b", "c"]
],
[
:then_this,
["b"]
]
]
How do I transform above so it looks like:
DoThisOperation.new(DoThatOperation.new(ThenThisOperation.new('b'), 'b' , 'c'), 'a')
This is as far as I've gotten:
require 'active_support/inflector'
class DoThisOperation
def initialize(successor = nil, a)
end
end
class DoThatOperation
def initialize(successor = nil, b, c)
end
end
class ThenThisOperation
def initialize(successor = nil, b)
end
end
operations = [
[
:do_this,
["a"]
],x
[
:do_that,
["b", "c"]
],
[
:then_this,
["b"]
]
]
operations.reverse.reduce do |result, element|
klass_name = element[0].to_s.camelize
args = element[1]
"#{klass_name}Operation".constantize.new(result, *args)
end
Is reduce/inject the right way to go about this? If so, what should I be doing above?
Is reduce/inject the right way to go about this?
Yes, but you need to pass an initial value to reduce, e.g. nil. Otherwise, the first element (in your case the last element) will be used as the initial value without being converted.
This would work:
operations.reverse.reduce(nil) do |result, element|
klass_name = element[0].to_s.camelize
args = element[1]
"#{klass_name}Operation".constantize.new(result, *args)
end
You can further simplify it by using array decomposition:
operations.reverse.reduce(nil) do |result, (name, args)|
klass_name = name.to_s.camelize
"#{klass_name}Operation".constantize.new(result, *args)
end
Or even:
operations.reverse.reduce(nil) do |result, (name, args)|
"#{name}_operation".camelize.constantize.new(result, *args)
end
I use the ExTwitter-library to poll data using a stream like this:
stream = ExTwitter.stream_sample(receive_messages: true)
for message <- stream do
case message do
tweet = %ExTwitter.Model.Tweet{} ->
IO.puts "tweet = #{tweet.text}"
deleted_tweet = %ExTwitter.Model.DeletedTweet{} ->
IO.puts "deleted tweet = #{deleted_tweet.status[:id]}"
limit = %ExTwitter.Model.Limit{} ->
IO.puts "limit = #{limit.track}"
stall_warning = %ExTwitter.Model.StallWarning{} ->
IO.puts "stall warning = #{stall_warning.code}"
_ ->
IO.inspect message
end
end
and it's working great but now I want to monitor the stream with a Supervisor. What is the simplest way to do that?
The simplest way would be to put this code in a function in a new module, add a start_link function that simply invokes this function through spawn_link, and add that module as a worker to your Supervisor. Here's a simple example:
defmodule M do
use Application
def start(_type, _args) do
import Supervisor.Spec, warn: false
children = [
worker(M.Streamer, []),
]
opts = [strategy: :one_for_one, name: M.Supervisor]
Supervisor.start_link(children, opts)
end
end
defmodule M.Streamer do
def start_link do
{:ok, spawn_link(__MODULE__, :main, [])}
end
def main do
IO.inspect(self)
for i <- Stream.cycle([1, 2, 3]) do
IO.puts i
:timer.sleep(1000)
end
end
end
Demo:
#PID<0.85.0>
iex(1)> 1
2
3
1
2
3
1
2
Process.exit3pid(0, 85, 0), :kill)
#PID<0.88.0>
1
true
iex(2)> 2
3
1
2
3
1
2
3
Process.exit(pid(0, 88, 0), :kill)
#PID<0.90.0>
true
1
iex(3)> 2
3
1
2
3
This might be a little hard to follow as the output was happening while I was typing, but all the PID values were printed whenever Streamer started, the Process.exit lines are the code entered by me, and true is the return value of those calls. As you can see, whenever I killed the M.Streamer process, it was restarted by the Supervisor.
I'm writing a gem for Rails that makes use of Arel. I have run into a case where the output generated by a subnode is different depending on how the output is generated. Below is a test case. Is this a bug or am I doing something wrong?
it 'should generate the same output' do
def ts_language; 'english'; end
def searchable_columns; [:id, :name]; end
def ts_column_sets
column_sets = if searchable_columns[0].is_a?(Array)
searchable_columns.map do |array|
array.map { |c| Table.new(:users)[c] }
end
else
searchable_columns.map { |c| Table.new(:users)[c] }.map { |x| [x] }
end
end
def ts_vectors
ts_column_sets.map do |columns|
coalesce = columns[1..-1].inject(columns[0]) do |memo, column|
Arel::Nodes::InfixOperation.new('||', memo, column)
end
coalesce = Arel::Nodes::InfixOperation.new('::', Arel::Nodes::NamedFunction.new('COALESCE', [coalesce, '']), Arel::Nodes::SqlLiteral.new('text'))
Arel::Nodes::NamedFunction.new('to_tsvector', [ts_language, coalesce])
end
end
def ts_query(query)
querytext = query.is_a?(Array) ? query.map(&:to_s).map(&:strip) : query.to_s.strip.split(" ")
querytext = querytext[1..-1].inject(querytext[0]) { |memo, c| memo + ' & ' + c }
querytext << ':*'
querytext = Arel::Nodes::InfixOperation.new('::', querytext, Arel::Nodes::SqlLiteral.new('text'))
Arel::Nodes::NamedFunction.new('to_tsquery', ['english', querytext])
end
node = Arel::Nodes::InfixOperation.new('##', ts_vectors[0], ts_query(0))
assert_equal 'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text)', node.left.to_sql
assert_equal 'to_tsquery(\'english\', \'0:*\' :: text)', node.right.to_sql
assert_equal 'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text) ## to_tsquery(0, \'0:*\' :: text)', node.to_sql
end
The line
assert_equal 'to_tsquery(\'english\', \'0:*\' :: text)', node.right.to_sql
is correct but the line
assert_equal 'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text) ## to_tsquery(0, \'0:*\' :: text)', node.to_sql
results in an error and the output is:
'to_tsvector(\'english\', COALESCE("users"."id", 0) :: text) ## to_tsquery(0, 0 :: text)'