I'm using ExTwitter Stream to track tweets in realtime and broadcast them via channel endpoint. I would like to create one process per event and assign to it one twitter stream listener, then when new subscriber join to the same event get previous stream state and also receive and broadcast new tweets.
How to create GenServer process from:
stream = ExTwitter.stream_filter(track: hashtags)
pid = spawn(fn ->
for tweet <- stream do
IO.puts tweet.text
MyApp.Endpoint.broadcast! "stream", "tweet", %{tweet: tweet.text}
end
end)
and assign it by event_id as child in following module:
defmodule MyApp.TwitterStream.Monitor do
require IEx
#moduledoc """
Store twitter stream per event_id
"""
use GenServer
def create(event_id, hashtags, coords) do
case GenServer.whereis(ref(event_id)) do
nil ->
Supervisor.start_child(MyApp.TwitterStream.Supervisor, [event_id, hashtags, coords])
_twitter_stream ->
IEx.pry
# return previous ExTwitter stream state and broadcast new tweets
{:error, :twitter_stream_already_exists}
end
end
def start_link(event_id, hashtags, coords) do
# stream = ExTwitter.stream_filter(track: hashtags)
# pid = spawn(fn ->
# for tweet <- stream do
# IO.puts tweet.text
# MyApp.Endpoint.broadcast! "stream", "tweet", %{tweet: tweet.text}
# end
# end)
GenServer.start_link(__MODULE__, %{hashtags: hashtags, coords: coords}, name: ref(event_id))
end
def stop(event_id) do
try_call(event_id, :stop)
end
def info(event_id) do
try_call(event_id, :info)
end
def handle_call(:stop, _from, state) do
# ExTwitter.stream_control(pid, :stop)
{:stop, :normal, :ok, state}
end
def handle_call(:info, _from, state) do
{:reply, state, state}
end
defp try_call(event_id, call_function) do
case GenServer.whereis(ref(event_id)) do
nil ->
{:error, :invalid_twitter_stream}
twitter_stream ->
GenServer.call(twitter_stream, call_function)
end
end
defp ref(event_id) do
{:global, {:twitter_stream, event_id}}
end
end
How to receive new tweets or eventually stop twitter stream outside the event Monitor?
Supervisor:
defmodule MyApp.TwitterStream.Supervisor do
use Supervisor
def start_link do
Supervisor.start_link(__MODULE__, :ok, name: __MODULE__)
end
def init(:ok) do
children = [
worker(MyApp.TwitterStream.Monitor, [], restart: :temporary)
]
supervise(children, strategy: :simple_one_for_one)
end
end
Related
I am having an issue with Handle Cast. After the module loads it calls periodic_comm which after every 5 seconds calls the save_data method. In my save_data method the logs are being printed but the statement does nothing.
GenServer.cast(__MODULE__, {:save_load_data, data})
The handle cast method is defined here
def handle_cast({:save_load_data, data}, state) do
Logger.error("uin saving data====================================")
# state = Map.put_new(state, :load_data, data)
# Logger.debug "updated state : #{inspect state}"
{:noreply, state}
end
The entire file looks like this
defmodule Core.Inverter.Manager do
# core inverter process that handles the inverter communication
use GenServer
require Logger
def start_link(_) do
Logger.info("putting some string")
GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
end
#impl true
def init(_) do
Logger.info("inverter Manager init'ed")
{:ok, %{configs: nil, inverter_connection: nil, inverter_impl: nil}}
end
# public api
# initialize the inverter and start periodic comm
def init_inverter(configs) do
GenServer.cast(__MODULE__, {:init_inverter, configs})
GenServer.cast(__MODULE__, {:start_periodic_comm, configs})
end
def inverter_off() do
GenServer.call(__MODULE__, :inverter_off)
end
#impl true
def handle_cast({:init_inverter, configs}, state) do
Logger.debug("initializing the inverter")
# fetching inverter type/configurations
inverter_impl = get_inverter_impl(configs)
# initializing inverter comm channel
inverter_connection = inverter_impl.init(configs)
state = %{
configs: configs,
inverter_connection: inverter_connection,
inverter_impl: inverter_impl
}
Logger.warn("inverter type #{state.inverter_impl}")
{:noreply, state}
end
def handle_cast({:start_periodic_comm, configs}, state) do
Logger.warn(
"Actually Starting inverter periodic COMM with inverter type #{state.inverter_impl}"
)
start_periodic_comm(state)
{:noreply, state}
end
#impl true
def handle_call(
:inverter_off,
_from,
%{inverter_impl: inverter_impl, inverter_connection: inverter_connection} = state
) do
{:ok, inverter_connection} = inverter_impl.inverter_off(inverter_connection)
{:reply, :ok, %{state | inverter_connection: inverter_connection}}
end
def handle_cast({:save_load_data, data}, state) do
Logger.error("uin saving data====================================")
# state = Map.put_new(state, :load_data, data)
# Logger.debug "updated state : #{inspect state}"
{:noreply, state}
end
defp start_periodic_comm(state) do
periodic_comm(state)
end
defp periodic_comm(state) do
# call functions from the inverter_type module, e.g infit.ex
receive do
after
5000 ->
{:ok, data} = state.inverter_impl.get_load_data(state)
save_data(data)
end
periodic_comm(state)
end
# defp save_data() do end
def save_data(data) do
Logger.debug("saving data")
Logger.debug("in savinf data data is : #{inspect(data)}")
GenServer.cast(__MODULE__, {:save_load_data, data})
end
defp get_inverter_impl(configs) do
# add case statements here
Core.Inverter.Infini
end
end
I'm not so familiar with elixir's syntax, but it seems to me that periodic_comm is looping:
defp periodic_comm(state) do
# call functions from the inverter_type module, e.g infit.ex
receive do
after
5000 ->
{:ok, data} = state.inverter_impl.get_load_data(state)
save_data(data)
end
periodic_comm(state)
end
regardless of the result of save_data, the thread is endlessly looping over calls to periodic_comm, so it gets no chance to receive and execute the save_load_data message.
In order to fix it, you should refactor the server to have the logic in handle_X without looping, which receive all kind of messages. You can use erlang:start_timer (I don't know the elixir's counterpart) or rely on the Timeout in the gen_server return values to receive timeout messages.
I'd like to be able to dynamically change a compute pipeline at runtime, but it seems that GenStage requires the compute graph to be defined at compile time through the subscribe_to: [...] mechanism. Is there a way to create dynamic compute graphs? For example in the below, I'd like to switch, at runtime, between the "subtract 7" and "subtract 4" vertices in my pipeline graph.
Is this possible using GenStage? I will likely have very complex pipelines so I need a solution that scales to changing graphs in complex ways, as opposed to ad-hoc solutions such as in this case, say, parameterising the integer to subtract. I'd like to be able to add or remove entire sub-trees, switch between subtrees, and add nodes into the graph including splicing them into the middle of any subtree including the main tree.
Please refer to EDIT further down
Here is the initial producer:
defmodule GenstageTest.Producer do
use GenStage
def start_link(initial \\ 1) do
GenStage.start_link(__MODULE__, initial, name: __MODULE__)
end
def init(counter), do: {:producer, counter}
def handle_demand(demand, state) do
events = Enum.to_list(state..(state + demand - 1))
{:noreply, events, state + demand}
end
end
Here is one of the producer_consumers:
defmodule GenstageTest.PcTimesFive do
use GenStage
def start_link do
GenStage.start_link(__MODULE__, :state_doesnt_matter, name: __MODULE__)
end
def init(state) do
{:producer_consumer, state, subscribe_to: [GenstageTest.PcAddOne]}
end
def handle_events(events, _from, state) do
numbers =
events
|> Enum.map(&(&1 * 5))
{:noreply, numbers, state}
end
end
and here is the final consumer:
defmodule GenstageTest.Consumer do
use GenStage
def start_link do
GenStage.start_link(__MODULE__, :state_doesnt_matter)
end
def init(state) do
{:consumer, state, subscribe_to: [GenstageTest.PcDivTwo]}
end
def handle_events(events, _from, state) do
for event <- events do
IO.inspect({self(), event, state})
end
# As a consumer we never emit events
{:noreply, [], state}
end
end
I
It is all modelled off the Elixir School Genstage tutorial.
All the modules and the mix.exs can be found on github.
EDIT 3 days later after partial answer from #AquarHEAD L.
I have managed to get runtime subscriptions working. Here are some modified producers, producer_consumers, and consumers respectively:
Producer:
defmodule GenstageTest.Producer do
use GenStage
def start_link(initial \\ 1) do
GenStage.start_link(__MODULE__, initial, name: __MODULE__)
end
def init(counter), do: {:producer, counter}
def handle_demand(demand, state) do
events = Enum.to_list(state..(state + demand - 1))
{:noreply, events, state + demand}
end
def handle_info({:doprint}, state) do
IO.puts "yep"
{:noreply, [], state}
end
def handle_info({:cancel, sublink}, state) do
GenStage.cancel sublink, []
{:noreply, [], state}
end
end
Producter_consumer:
defmodule GenstageTest.PcAddOne do
use GenStage
def start_link do
GenStage.start_link(__MODULE__, :state_doesnt_matter, name: __MODULE__)
end
def init(state) do
{:producer_consumer, state}
end
def handle_events(events, _from, state) do
numbers =
events
|> Enum.map(&(&1 + 1))
{:noreply, numbers, state}
end
end
Consumer:
defmodule GenstageTest.Consumer do
use GenStage
def start_link do
GenStage.start_link(__MODULE__, :state_doesnt_matter)
end
def init(state) do
{:consumer, state}
end
def handle_events(events, _from, state) do
for event <- events do
IO.inspect event
#File.write("/home/tbrowne/scratch/output.txt",
# Kernel.inspect(event) <> " ", [:append])
:timer.sleep(100)
end
# As a consumer we never emit events
{:noreply, [], state}
end
end
Now once these are all available in the lib directory (remember to add {:gen_stage, "~> 0.11"} to your mix.exs deps), or copied and pasted into IEX, then the following will work perfectly:
{:ok, p} = GenstageTest.Producer.start_link(0)
{:ok, a1} = GenstageTest.PcAddOne.start_link()
{:ok, c} = GenstageTest.Consumer.start_link()
{:ok, link1} = GenStage.sync_subscribe(a1, to: p, min_demand: 0, max_demand: 1, cancel: :transient)
{:ok, link2} = GenStage.sync_subscribe(c, to: a1, min_demand: 0, max_demand: 1, cancel: :transient)
The issue now is though, that I still don't know how to cancel the subscription. There is a cancel function and there is also a stop function. GenStage.stop(c) for example seems to do nothing, while my various attempts at GenStage.cancel/3 only give errors.
To recap, what I need now is to be able to stop certain stages and replace them with others. What is the syntax for cancelling a subcsription, and from where is it called? It is not well explained in the docs as there is no concrete example.
You can absolutely change the pipeline at runtime, checkout the first example in GenStage documentation, you can also use the :manual mode to fine control the demand. There's also API to cancel subscription. I think these are enough to dynamically manage GenStage pipelines.
Why not implement your own GenStage.Dispatcher? Here is behaviour
I am trying to show a post to the first friend of a person first and other's after making a delay of 1 min. For that I am using GenServer.
The problem is that the first friend as well as the other friends are getting the post after 1 min.
Here is my code of GenServer:
defmodule Phoenix.SchedulePost do
use GenServer
def start_link(state) do
GenServer.start_link(__MODULE__, state)
end
def init(state) do
schedule_post(state)
{:ok, state}
end
# handling looby
def handle_info(:postSchedule, state) do
#sending posts to others
{:noreply, state}
end
# scheduling a task
defp schedule_post(state) do
IO.puts "scheduling the task"
Process.send_after(self(), :postSchedule, 60*1000)
end
end
I am starting a GenServer process for each post request and sending it to the first friend. Here is the code:
def handle_in("post:toFrstFrnd", %{"friendId"=>friendId,"body" => body}, socket) do
newSocket = PhoenixWeb.SocketBucket.get_socket(friendId)
if newSocket != nil do
push newSocket, "post:toFrstFrnd", %{"friendId": friendId,"body": body}
end
Phoenix.SchedulePost.start_link(postId)
{:noreply, socket}
end
Help me out, thank you in advance.
Note: I know it's a rather old question, but maybe someone else has a similar problem and ends up here.
I think you want to trigger an action initially and then another action a minute later. The problem with your code is that you call the schedule_post method in your init method and it does nothing for a minute. After one minute you send a message to the process itself, whereupon handle_info method takes over. But now it is already much too late for the initial action.
Here is an example how you could do it:
defmodule Phoenix.SchedulePost do
use GenServer
def start_link(state \\ []) do
GenServer.start_link(__MODULE__, state)
end
def init(state) do
send(self(), :first_action)
{:ok, state}
end
def handle_info(:first_action, state) do
IO.puts("Called immediately")
# Do something here...
Process.send_after(self(), :second_action, 60 * 1000)
{:noreply, state}
end
def handle_info(:second_action, state) do
IO.puts "Called after 1 min"
# Do something here...
{:noreply, state}
end
end
But keep in mind that the process will continue to live even after the second action is done. It will not terminate automatically, you have to take care of that.
Has something changed with the way Erlang 20/ Elixir 1.5 handle file paths to network shared folders? In my Elixir app running on a Windows server, I used to be able to refer to file paths that include virtual drive names (e.g., V:) which is mapped to shared network folders (eg., \server1\shared_folder), but after upgrading Erlang/Elixir recently (it could be a coincidence) it doesn't seem to recognize these file paths.
For instance, Monitor.start_link(dirs: ["C:/data/"]) works, but Monitor.start_link(dirs: ["V:/data/"] doesn't.
In case it helps, here is the Monitor module.
defmodule FinReporting.Monitor do
use GenServer
alias FinReporting.Repo
alias FinReporting.AppState
def start_link(args) do
GenServer.start_link(__MODULE__, args)
end
def init(args) do
{:ok, watcher_pid} = FileSystem.start_link(args)
FileSystem.subscribe(watcher_pid)
IO.inspect(watcher_pid, label: "Watcher PID")
{:ok, %{watcher_pid: watcher_pid}}
end
def handle_info({:file_event, watcher_pid, {path, events}},
%{watcher_pid: watcher_pid}=state) do
callback(path, events)
{:noreply, state}
end
def handle_info({:file_event, watcher_pid, :stop},
%{watcher_pid: watcher_pid}=state) do
callback(:stop)
{:noreply, state}
end
# The callback runs a python module using erlport
def callback(:stop) do
IO.puts "STOP"
end
def callback(file_path, events) do
case {file_path, events} do
{"e:\\reporting_temp\\data\\mls_alltrans\\AllTrans - All MLS_postdate.txt", [:modified]} ->
affix = File.stat!(file_path).mtime
|> :calendar.universal_time_to_local_time
|> Ecto.DateTime.from_erl()
|> Calendar.Strftime.strftime!("%b %d (%a) %l:%M %P")
result = AppState.changeset(%AppState{}, %{key: "alltrans_filechange_" <> affix, value: "downloaded"})
|> Repo.insert
case result do
{:ok, struct} ->
IO.inspect("inserted successfully")
py_mls_upload()
{:error, changeset} -> IO.inspect("insert failed")
end
{ _ , _ } ->
IO.inspect {file_path, events, "irrelevant"}
end
end
defp py_mls_upload do
{:ok, pp} = :python.start([{:python_path, to_char_list(Path.expand("lib/python_scripts"))}, {:python, 'python'}])
:python.call(pp, :mlsupload, :run, [])
end
end
I am new to elixir and OTP.
I want to use GenServer.Behaviour for my Server, but for some Reasons elixir can't find it. I have created a mix project, but when I type mix compile I get the following error
== Compilation error on file lib/lecture3.ex ==
** (CompileError) lib/lecture3.ex:2: module GenServer.Behaviour is not loaded and could not be found
(elixir) expanding macro: Kernel.use/1
lib/lecture3.ex:2: Cache (module)
I guess I have to include the module, but how?
mix.exs:
defmodule LECTURE3.Mixfile do
use Mix.Project
def project do
[app: :lecture3,
version: "0.1.0",
elixir: "~> 1.3",
build_embedded: Mix.env == :prod,
start_permanent: Mix.env == :prod,
deps: deps()]
end
def application do
[applications: [:logger]]
end
defp deps do
[]
end
end
lecture3.ex:
defmodule Cache do
use GenServer.Behaviour
def handle_cast({:put, url, page}, {pages, size}) do
new_pages = Dict.put(pages, url, page)
new_size = size + byte_size(page)
{:noreply, {new_pages, new_size}}
end
def handle_call({:get, url}, _from, {pages, size}) do
{:reply, pages[url], {pages, size}}
end
def handle_call({:size}, _from, {pages, size}) do
{:reply, size, {pages, size}}
end
def start_link do
:gen_server.start_link({:local,:cache}, __MODULE__, {HashDict.new, 0}, [])
end
def put(url, page) do
:gen_server.cast(:cache, {:put, url, page})
end
def get(url) do
:gen_server.call(:cache, {:get, url})
end
def size do
:gen_server.call(:cache, {:size})
end
end
defmodule CacheSupervisor do
def init(_args) do
workers = [worker(Cache, [])]
supervise(workers, strategy: :one_for_one)
end
def start_link(domain) do
:supervisor.start_link(__MODULE__, [domain])
end
end
Enum.map(["de","edu", "com" ,"it"], fn(x)-> CacheSupervisor.start_link(x)
end)
Actually GenServer is behavour, so try simply use GenServer. GenServer in Elixir is wrapper for gen_server in Erlang and it provides defaults for undefined functions (so in Erlang you have to always defined 6 functions and in Elixir not).
You don't have to use explicitly gen_server, which is Erlang's module, but use GenServer. Check this out.