Suppose I have the following data:
var1,var2,var3
0.942856823,0.568425866,0.325885379
1.227681099,1.335672206,0.925331054
1.952671045,1.829479996,1.512280854
2.45428731,1.990174152,1.534456808
2.987783477,2.78975186,1.725095748
3.651682331,2.966399127,1.972274564
3.768010479,3.211381506,1.993080807
4.509429614,3.642983433,2.541071547
4.81498729,3.888415006,3.218031802
Here is the code:
open System.IO
open MathNet.Numerics.LinearAlgebra
let rows = [|for line in File.ReadAllLines("Z:\\mypath.csv")
|> Seq.skip 1 do yield line.Split(',') |> Array.map float|]
let data = DenseMatrix.ofRowArrays rows
let data_logdiff =
DenseMatrix.init (data.RowCount-1) (data.ColumnCount)
(fun j i -> if j = 0 then 0. else data.At(j, i) / data.At(j-1, i) |> log)
let alpha = vector [for i in data_logdiff.EnumerateColumns() -> i |> Statistics.Mean]
let sigsq (values:Vector<float>) (avg: float) =
let sqr x = x * x
let result = values |> (fun i -> sqr (i - avg))
result
sigsq (data_logdiff.Column(i), alpha.[0]) |> printfn "%A"
Error: The type ''a * 'b' is not compatible with the type 'Vector<float>'
This is all for a broadcast operation between a matrix and a vector. All these acrobatics to do a simple mean((y-alpha).^2) in MATLAB.
You have a mistake in your code, and the F# compiler complains about it, albeit in a somewhat obscure way. You define your function:
let sigsq (values:Vector<float>) (avg: float) =
This is a function that takes two arguments. (Actually it's a function taking one argument, returning another function taking one argument.) But you call it like this:
sigsq (data_logdiff.Column(i), alpha.[0]) |> printfn "%A"
You tuple the arguments, and for F# functions (a,b) is one argument, which is a tuple. You should call your function like this:
sigsq (data_logdiff.Column(0)) (alpha.[0])
or
sigsq <| data_logdiff.Column(0) <| alpha.[0]
and my favorite one:
data_logdiff.Column(0) |> sigsq <| alpha.[0]
I replaced the (i) with 0 in your code. You can map through the columns if you want to loop:
data_logdiff.EnumerateColumnsIndexed() |> Seq.map (fun (i,col) -> sigsq col alpha.[i])
I'm trying to implement Kosaraju's algorithm on a large graph
as part of an assignment [MOOC Algo I Stanford on Coursera]
https://en.wikipedia.org/wiki/Kosaraju%27s_algorithm
The current code works on a small graph, but I'm hitting Stack Overflow during runtime execution.
Despite having read the relevant chapter in Expert in F#, or other available examples on websites and SO, i still don't get how to use continuation to solve this problem
Below is the full code for general purpose, but it will already fail when executing DFSLoop1 and the recursive function DFSsub inside. I think I'm not making the function tail recursive [because of the instructions
t<-t+1
G.[n].finishingtime <- t
?]
but i don't understand how i can implement the continuation properly.
When considering only the part that fails, DFSLoop1 is taking as argument a graph to which we will apply Depth-First Search. We need to record the finishing time as part of the algo to proceed to the second part of the algo in a second DFS Loop (DFSLoop2) [of course we are failing before that].
open System
open System.Collections.Generic
open System.IO
let x = File.ReadAllLines "C:\Users\Fagui\Documents\GitHub\Learning Fsharp\Algo Stanford I\PA 4 - SCC.txt";;
// let x = File.ReadAllLines "C:\Users\Fagui\Documents\GitHub\Learning Fsharp\Algo Stanford I\PA 4 - test1.txt";;
// val x : string [] =
let splitAtTab (text:string)=
text.Split [|'\t';' '|]
let splitIntoKeyValue (A: int[]) =
(A.[0], A.[1])
let parseLine (line:string)=
line
|> splitAtTab
|> Array.filter (fun s -> not(s=""))
|> Array.map (fun s-> (int s))
|> splitIntoKeyValue
let y =
x |> Array.map parseLine
//val it : (int * int) []
type Children = int[]
type Node1 =
{children : Children ;
mutable finishingtime : int ;
mutable explored1 : bool ;
}
type Node2 =
{children : Children ;
mutable leader : int ;
mutable explored2 : bool ;
}
type DFSgraphcore = Dictionary<int,Children>
let directgraphcore = new DFSgraphcore()
let reversegraphcore = new DFSgraphcore()
type DFSgraph1 = Dictionary<int,Node1>
let reversegraph1 = new DFSgraph1()
type DFSgraph2 = Dictionary<int,Node2>
let directgraph2 = new DFSgraph2()
let AddtoGraph (G:DFSgraphcore) (n,c) =
if not(G.ContainsKey n) then
let node = [|c|]
G.Add(n,node)
else
let c'= G.[n]
G.Remove(n) |> ignore
G.Add (n, Array.append c' [|c|])
let inline swaptuple (a,b) = (b,a)
y|> Array.iter (AddtoGraph directgraphcore)
y|> Array.map swaptuple |> Array.iter (AddtoGraph reversegraphcore)
for i in directgraphcore.Keys do
if reversegraphcore.ContainsKey(i) then do
let node = {children = reversegraphcore.[i] ;
finishingtime = -1 ;
explored1 = false ;
}
reversegraph1.Add (i,node)
else
let node = {children = [||] ;
finishingtime = -1 ;
explored1 = false ;
}
reversegraph1.Add (i,node)
directgraphcore.Clear |> ignore
reversegraphcore.Clear |> ignore
// for i in reversegraph1.Keys do printfn "%d %A" i reversegraph1.[i].children
printfn "pause"
Console.ReadKey() |> ignore
let num_nodes =
directgraphcore |> Seq.length
let DFSLoop1 (G:DFSgraph1) =
let mutable t = 0
let mutable s = -1
let mutable k = num_nodes
let rec DFSsub (G:DFSgraph1)(n:int) (cont:int->int) =
//how to make it tail recursive ???
G.[n].explored1 <- true
// G.[n].leader <- s
for j in G.[n].children do
if not(G.[j].explored1) then DFSsub G j cont
t<-t+1
G.[n].finishingtime <- t
// end of DFSsub
for i in num_nodes .. -1 .. 1 do
printfn "%d" i
if not(G.[i].explored1) then do
s <- i
( DFSsub G i (fun s -> s) ) |> ignore
// printfn "%d %d" i G.[i].finishingtime
DFSLoop1 reversegraph1
printfn "pause"
Console.ReadKey() |> ignore
for i in directgraphcore.Keys do
let node = {children =
directgraphcore.[i]
|> Array.map (fun k -> reversegraph1.[k].finishingtime) ;
leader = -1 ;
explored2= false ;
}
directgraph2.Add (reversegraph1.[i].finishingtime,node)
let z = 0
let DFSLoop2 (G:DFSgraph2) =
let mutable t = 0
let mutable s = -1
let mutable k = num_nodes
let rec DFSsub (G:DFSgraph2)(n:int) (cont:int->int) =
G.[n].explored2 <- true
G.[n].leader <- s
for j in G.[n].children do
if not(G.[j].explored2) then DFSsub G j cont
t<-t+1
// G.[n].finishingtime <- t
// end of DFSsub
for i in num_nodes .. -1 .. 1 do
if not(G.[i].explored2) then do
s <- i
( DFSsub G i (fun s -> s) ) |> ignore
// printfn "%d %d" i G.[i].leader
DFSLoop2 directgraph2
printfn "pause"
Console.ReadKey() |> ignore
let table = [for i in directgraph2.Keys do yield directgraph2.[i].leader]
let results = table |> Seq.countBy id |> Seq.map snd |> Seq.toList |> List.sort |> List.rev
printfn "%A" results
printfn "pause"
Console.ReadKey() |> ignore
Here is a text file with a simple graph example
1 4
2 8
3 6
4 7
5 2
6 9
7 1
8 5
8 6
9 7
9 3
(the one which is causing overflow is 70Mo big with around 900,000 nodes)
EDIT
to clarify a few things first
Here is the "pseudo code"
Input: a directed graph G = (V,E), in adjacency list representation. Assume that the vertices V are labeled
1, 2, 3, . . . , n.
1. Let Grev denote the graph G after the orientation of all arcs have been reversed.
2. Run the DFS-Loop subroutine on Grev, processing vertices according to the given order, to obtain a
finishing time f(v) for each vertex v ∈ V .
3. Run the DFS-Loop subroutine on G, processing vertices in decreasing order of f(v), to assign a leader
to each vertex v ∈ V .
4. The strongly connected components of G correspond to vertices of G that share a common leader.
Figure 2: The top level of our SCC algorithm. The f-values and leaders are computed in the first and second
calls to DFS-Loop, respectively (see below).
Input: a directed graph G = (V,E), in adjacency list representation.
1. Initialize a global variable t to 0.
[This keeps track of the number of vertices that have been fully explored.]
2. Initialize a global variable s to NULL.
[This keeps track of the vertex from which the last DFS call was invoked.]
3. For i = n downto 1:
[In the first call, vertices are labeled 1, 2, . . . , n arbitrarily. In the second call, vertices are labeled by
their f(v)-values from the first call.]
(a) if i not yet explored:
i. set s := i
ii. DFS(G, i)
Figure 3: The DFS-Loop subroutine.
Input: a directed graph G = (V,E), in adjacency list representation, and a source vertex i ∈ V .
1. Mark i as explored.
[It remains explored for the entire duration of the DFS-Loop call.]
2. Set leader(i) := s
3. For each arc (i, j) ∈ G:
(a) if j not yet explored:
i. DFS(G, j)
4. t + +
5. Set f(i) := t
Figure 4: The DFS subroutine. The f-values only need to be computed during the first call to DFS-Loop, and
the leader values only need to be computed during the second call to DFS-Loop.
EDIT
i have amended the code, with the help of an experienced programmer (a lisper but who has no experience in F#) simplifying somewhat the first part to have more quickly an example without bothering about non-relevant code for this discussion.
The code focuses only on half of the algo, running DFS once to get finishing times of the reversed tree.
This is the first part of the code just to create a small example
y is the original tree. the first element of a tuple is the parent, the second is the child. But we will be working with the reverse tree
open System
open System.Collections.Generic
open System.IO
let x = File.ReadAllLines "C:\Users\Fagui\Documents\GitHub\Learning Fsharp\Algo Stanford I\PA 4 - SCC.txt";;
// let x = File.ReadAllLines "C:\Users\Fagui\Documents\GitHub\Learning Fsharp\Algo Stanford I\PA 4 - test1.txt";;
// val x : string [] =
let splitAtTab (text:string)=
text.Split [|'\t';' '|]
let splitIntoKeyValue (A: int[]) =
(A.[0], A.[1])
let parseLine (line:string)=
line
|> splitAtTab
|> Array.filter (fun s -> not(s=""))
|> Array.map (fun s-> (int s))
|> splitIntoKeyValue
// let y =
// x |> Array.map parseLine
//let y =
// [|(1, 4); (2, 8); (3, 6); (4, 7); (5, 2); (6, 9); (7, 1); (8, 5); (8, 6);
// (9, 7); (9, 3)|]
// let y = Array.append [|(1,1);(1,2);(2,3);(3,1)|] [|for i in 4 .. 10000 do yield (i,4)|]
let y = Array.append [|(1,1);(1,2);(2,3);(3,1)|] [|for i in 4 .. 99999 do yield (i,i+1)|]
//val it : (int * int) []
type Children = int list
type Node1 =
{children : Children ;
mutable finishingtime : int ;
mutable explored1 : bool ;
}
type Node2 =
{children : Children ;
mutable leader : int ;
mutable explored2 : bool ;
}
type DFSgraphcore = Dictionary<int,Children>
let directgraphcore = new DFSgraphcore()
let reversegraphcore = new DFSgraphcore()
type DFSgraph1 = Dictionary<int,Node1>
let reversegraph1 = new DFSgraph1()
let AddtoGraph (G:DFSgraphcore) (n,c) =
if not(G.ContainsKey n) then
let node = [c]
G.Add(n,node)
else
let c'= G.[n]
G.Remove(n) |> ignore
G.Add (n, List.append c' [c])
let inline swaptuple (a,b) = (b,a)
y|> Array.iter (AddtoGraph directgraphcore)
y|> Array.map swaptuple |> Array.iter (AddtoGraph reversegraphcore)
// définir reversegraph1 = ... with....
for i in reversegraphcore.Keys do
let node = {children = reversegraphcore.[i] ;
finishingtime = -1 ;
explored1 = false ;
}
reversegraph1.Add (i,node)
for i in directgraphcore.Keys do
if not(reversegraphcore.ContainsKey(i)) then do
let node = {children = [] ;
finishingtime = -1 ;
explored1 = false ;
}
reversegraph1.Add (i,node)
directgraphcore.Clear |> ignore
reversegraphcore.Clear |> ignore
// for i in reversegraph1.Keys do printfn "%d %A" i reversegraph1.[i].children
printfn "pause"
Console.ReadKey() |> ignore
let num_nodes =
directgraphcore |> Seq.length
So basically the graph is (1->2->3->1)::(4->5->6->7->8->....->99999->10000)
and the reverse graph is (1->3->2->1)::(10000->9999->....->4)
here is the main code written in direct style
//////////////////// main code is below ///////////////////
let DFSLoop1 (G:DFSgraph1) =
let mutable t = 0
let mutable s = -1
let rec iter (n:int) (f:'a->unit) (list:'a list) : unit =
match list with
| [] -> (t <- t+1) ; (G.[n].finishingtime <- t)
| x::xs -> f x ; iter n f xs
let rec DFSsub (G:DFSgraph1) (n:int) : unit =
let my_f (j:int) : unit = if not(G.[j].explored1) then (DFSsub G j)
G.[n].explored1 <- true
iter n my_f G.[n].children
for i in num_nodes .. -1 .. 1 do
// printfn "%d" i
if not(G.[i].explored1) then do
s <- i
DFSsub G i
printfn "%d %d" i G.[i].finishingtime
// End of DFSLoop1
DFSLoop1 reversegraph1
printfn "pause"
Console.ReadKey() |> ignore
its not tail recursive, so we use continuations, here is the same code adapted to CPS style:
//////////////////// main code is below ///////////////////
let DFSLoop1 (G:DFSgraph1) =
let mutable t = 0
let mutable s = -1
let rec iter_c (n:int) (f_c:'a->(unit->'r)->'r) (list:'a list) (cont: unit->'r) : 'r =
match list with
| [] -> (t <- t+1) ; (G.[n].finishingtime <- t) ; cont()
| x::xs -> f_c x (fun ()-> iter_c n f_c xs cont)
let rec DFSsub (G:DFSgraph1) (n:int) (cont: unit->'r) : 'r=
let my_f_c (j:int)(cont:unit->'r):'r = if not(G.[j].explored1) then (DFSsub G j cont) else cont()
G.[n].explored1 <- true
iter_c n my_f_c G.[n].children cont
for i in maxnum_nodes .. -1 .. 1 do
// printfn "%d" i
if not(G.[i].explored1) then do
s <- i
DFSsub G i id
printfn "%d %d" i G.[i].finishingtime
DFSLoop1 reversegraph1
printfn "faré"
printfn "pause"
Console.ReadKey() |> ignore
both codes compile and give the same results for the small example (the one in comment) or the same tree that we are using , with a smaller size (1000 instead of 100000)
so i don't think its a bug in the algo here, we've got the same tree structure, just a bigger tree is causing problems. it looks to us the continuations are well written. we've typed the code explicitly. and all calls end with a continuation in all cases...
We are looking for expert advice !!! thanks !!!
I did not try to understand the whole code snippet, because it is fairly long, but you'll certainly need to replace the for loop with an iteration implemented using continuation passing style. Something like:
let rec iterc f cont list =
match list with
| [] -> cont ()
| x::xs -> f x (fun () -> iterc f cont xs)
I didn't understand the purpose of cont in your DFSub function (it is never called, is it?), but the continuation based version would look roughly like this:
let rec DFSsub (G:DFSgraph2)(n:int) cont =
G.[n].explored2 <- true
G.[n].leader <- s
G.[n].children
|> iterc
(fun j cont -> if not(G.[j].explored2) then DFSsub G j cont else cont ())
(fun () -> t <- t + 1)
Overflowing the stack when you recurse through hundreds of thousands of entries isn't bad at all, really. A lot of programming language implementations will choke on much shorter recursions than that. You're having serious programmer problems — nothing to be ashamed of!
Now if you want to do deeper recursions than your implementation will handle, you need to transform your algorithm so it is iterative and/or tail-recursive (the two are isomorphic — except that tail-recursion allows for decentralization and modularity, whereas iteration is centralized and non-modular).
To transform an algorithm from recursive to tail-recursive, which is an important skill to possess, you need to understand the state that is implicitly stored in a stack frame, i.e. those free variables in the function body that change across the recursion, and explicitly store them in a FIFO queue (a data structure that replicates your stack, and can be implemented trivially as a linked list). Then you can pass that linked list of reified frame variables as an argument to your tail recursive functions.
In more advanced cases where you have many tail recursive functions each with a different kind of frame, instead of simple self-recursion, you may need to define some mutually recursive data types for the reified stack frames, instead of using a list. But I believe Kosaraju's algorithm only involves self-recursive functions.
OK, so the code given above was the RIGHT code !
the problem lies with the compiler of F#
here is some words about it from Microsoft
http://blogs.msdn.com/b/fsharpteam/archive/2011/07/08/tail-calls-in-fsharp.aspx
Basically, be careful with the settings, in default mode, the compiler may NOT make automatically the tail calls. To do so, in VS2015, go to the Solution Explorer, right click with the mouse and click on "Properties" (the last element of the scrolling list)
Then in the new window, click on "Build" and tick the box "Generate tail calls"
It is also to check if the compiler did its job looking at the disassembly using
ILDASM.exe
you can find the source code for the whole algo in my github repository
https://github.com/FaguiCurtain/Learning-Fsharp/blob/master/Algo%20Stanford/Algo%20Stanford/Kosaraju_cont.fs
on a performance point of view, i'm not very satisfied. The code runs on 36 seconds on my laptop. From the forum with other fellow MOOCers, C/C++/C# typically executes in subsecond to 5s, Java around 10-15, Python around 20-30s.
So my implementation is clearly not optimized. I am now happy to hear about tricks to make it faster !!! thanks !!!!
I am attempting to generate a series of guesses for the second Taxicab number. What I want to do is is call the Attempt function on a series of integers in a finite sequence. I have my two questions about implementation in the comments.
A taxi cab number, in case your wondering, is the least number that satisfied the sum of 2 unique cubes in for n unique sets of 2 unique cubes. Ta(2) is 1729.
[<EntryPoint>]
let main argv =
let Attempt (start : int) =
let stop = start+20
let integerList = [start..stop]
let list = List.init 3 (fun x -> integerList.[x])
//Is there a simple way to make initialize the list with random indices of integerList?
let Cube x = x*x*x
let newlist = list |> List.map (fun x -> Cube x)
let partitionList (x : List<int>) (y : int) = List.sum [x.[y];x.[y+1]]
let intLIST = [0..2]
let partitionList' = [for i in intLIST do yield partitionList newlist i]
let x = Set.ofList partitionList'
let y = Set.ofList partitionList'
//I was going to try to use some kind of equality operator to determine whether the two sets were equal, which could tell me whether we had actually found a Taxicab number by the weakened definition.
System.Console.Write(list)
System.Console.Write(newlist)
let rnd = System.Random()
//My primary question is how can I convert a random to an integer to use in start for the function Attempt?
System.Console.ReadKey() |> ignore
printfn("%A") argv
0
Dirty way to initialize list with random indexes of another list:
let randomIndexes count myList =
let rand = System.Random()
seq {
for n = 1 to count do
yield rand.Next(List.length myList) }
|> Seq.distinct
//|> Seq.sort // if you need them sorted
|> List.ofSeq
let result = randomIndexes 5 [3;2;4;5]
printfn "%A" result
I am prototyping how I am going to handle Double.NaN values in an F# array, and the first step, trying to simply count how many there are, has me stumped. The value "howMany" comes back as zero in my code, but I know there are 2, because I set 2 value to be Double.NaN. Can anyone point out what I am missing? Thanks!
let rnd = new System.Random()
let fakeAlphas = Array.init 10 (fun _ -> rnd.NextDouble());;
fakeAlphas.[0] <- Double.NaN;
fakeAlphas.[1] <- Double.NaN;
let countNA arr = arr |> Array.filter (fun x -> x = Double.NaN) |> Array.length;;
let howMany = countNA fakeAlphas;;
To answer the general question in the title:
let HowManySatisfy pred = Seq.filter pred >> Seq.length
for example
let nums = [1;2;3;4;5]
let countEvens = nums |> HowManySatisfy (fun n -> n%2=0)
printfn "%d" countEvens
Double.NaN = n is false for all n. See the MSDN page for Double.NaN.
Instead use Double.IsNaN. See the MSDN page for more information.
I think you need to use the Double.IsNan method. So your filter function would be:
(fun x -> Double.IsNan x)
I believe the issue is that NaN never equals anything -- even another NaN!
let aBunch = 1000
let offset = 0
let getIt offset =
MyIEnumerable
|> Seq.skip aBunch * offset
|> Seq.take aBunch
|> Seq.iter ( .. some processing ...)
Calling getIt() with different offsets eventually gives me an 'Invalid operation' exception with additional info that 'the input sequence had insufficient elements'
I try to understand why, as both the Seq.Skip and Seq.take do not generate an exception according to the online documentation FSharp Collections
Version: (Visual Studio 2010) Beta 1
I know this is an old question, but in case someone comes across this in a search the way I did:
You can use Seq.truncate if you want at most n items. It won't throw an exception if fewer than n items are available.
Both Seq.skip and Seq.take will throw this exception if called with a value larger than the sequence. You can check the source code in Seq.fs to see why:
let skip count (sequence: seq<_>) =
{ use e = sequence.GetEnumerator()
let latest = ref (Unchecked.defaultof<_>)
let ok = ref false
for i in 1 .. count do
if not (e.MoveNext()) then
raise <| System.InvalidOperationException "the input sequence had insufficient elements"
while e.MoveNext() do
yield e.Current }
let take count (sequence : seq<'T>) =
if count < 0 then invalidArg "count" "the number of elements to take may not be negative"
(* Note: don't create or dispose any IEnumerable if n = 0 *)
if count = 0 then empty else
{ use e = sequence.GetEnumerator()
for i in 0 .. count - 1 do
if not (e.MoveNext()) then
raise <| System.InvalidOperationException "the input sequence had insufficient elements"
yield e.Current }
For an exceptionless skip you can add your own version to the Seq module like this:
module Seq =
let skipSafe (num: int) (source: seq<'a>) : seq<'a> =
seq {
use e = source.GetEnumerator()
let idx = ref 0
let loop = ref true
while !idx < num && !loop do
if not(e.MoveNext()) then
loop := false
idx := !idx + 1
while e.MoveNext() do
yield e.Current
}
Combined with Seq.truncate (which is an exceptionless Seq.take equivalent - it will take as much items are available without throwing an exception).
[1..10]
|> Seq.skipSafe 20
|> Seq.truncate 5
(* returns empty seq *)
Here's a slightly shorter "skipSafe" implementation using built in functions:
module Seq =
let skipSafe num =
Seq.zip (Seq.initInfinite id)
>> Seq.skipWhile (fun (i, _) -> i < num)
>> Seq.map snd
Or if you wish to just inline it into your current pipeline directly, replace
|> Seq.skip num
with
|> Seq.zip (Seq.initInfinite id)
|> Seq.skipWhile (fun (i, _) -> i < num)
|> Seq.map snd
module Seq =
let trySkip count source =
source |> Seq.indexed |> Seq.filter(fst >> (<=) count) |> Seq.map snd