Download Windows Phone app reviews using F# - f#

Question

I am learning F# on my own (this is for fun, it is not for work/school) and I am trying to write a simple parser which count the number of reviews across multiple markets for a Windows Phone app. There's no doubt that the code I have so far is ugly, but I am trying to improve it and follow functional programming paradigm. Since I come from the C, C++, C# world, it is pretty hard.
Coming from C world, I like null values. I know that functional programming / F# doesn't encourage the use of null, but I can't figure out a way to not use it. For example, in the function parse there's a null check. How do I not do that?
Right now my code only count the number of reviews on the first page, but it is possible that an app has more than 10 reviews and as a result multiple pages. How do I recursively go through all page (functuion downloadReviews or parse).
How could we extend this code to be entirely async?
Below is the code I have so far. In addition to the questions above, I would really like if someone could help me and give me directions on how to improve the overall structure of my code.
open System
open System.IO
open System.Xml
open System.Xml.Linq
open Printf
type DownloadPageResult = {
Uri: System.Uri;
ErrorOccured: bool;
Source: string;
}
type ReviewData = {
CurrentPageUri: System.Uri;
NextPageUri: System.Uri;
NumberOfReviews: int;
}
module ReviewUrl =
let getBaseUri path =
new Uri(sprintf "http://cdn.marketplaceedgeservice.windowsphone.com/%s" path)
let getUri country locale appId =
getBaseUri(sprintf "/v8/ratings/product/%s/reviews?os=8.0.0.0&cc=%s&oc=&lang=%s&hw=520170499&dm=Test&chunksize=10" appId country locale)
let downloadPage (uri: System.Uri) =
try
use webClient = new System.Net.WebClient()
printfn "%s" (uri.ToString())
webClient.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
webClient.Headers.Add("Accept-Encoding", "zip,deflate,sdch")
webClient.Headers.Add("Accept-Language", "en-US,en;q=0.8,fr;q=0.6")
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1482.0 Safari/537.36")
{ Uri = uri; Source = webClient.DownloadString(uri); ErrorOccured = false }
with error -> { Uri = uri; Source = String.Empty; ErrorOccured = true }
let downloadReview country locale appId =
let uri = ReviewUrl.getUri country locale appId
downloadPage uri
let parse(pageResult: DownloadPageResult) =
if pageResult.ErrorOccured then { CurrentPageUri = pageResult.Uri; NextPageUri = null; NumberOfReviews = 0 }
else
let reader = new StringReader(pageResult.Source)
let doc = XDocument.Load(reader)
let ns = XNamespace.Get("http://www.w3.org/2005/Atom")
let nextUrl = query { for link in doc.Descendants(ns + "link") do
where (link.Attribute(XName.Get("rel")).Value = "next")
select link.Value
headOrDefault }
if nextUrl = null then
{ CurrentPageUri = pageResult.Uri; NextPageUri = null; NumberOfReviews = doc.Descendants(ns + "entry") |> Seq.length }
else
{ CurrentPageUri = pageResult.Uri; NextPageUri = ReviewUrl.getBaseUri(nextUrl); NumberOfReviews = doc.Descendants(ns + "entry") |> Seq.length }
let downloadReviews(locale: string) =
let appId = "4e08377c-1240-4f80-9c35-0bacde2c66b6"
let country = locale.Substring(3)
let pageResult = downloadReview country locale appId
let parseResult = parse pageResult
parseResult
[<EntryPoint>]
let main argv =
let locales = [| "en-US"; "en-GB"; |]
let results = locales |> Array.map downloadReviews
printfn "%A" results
0

Answer 1

I was playing with this problem a bit more and tried using the XML type provider and other features from F# Data. It is not complete code, but it should be enough to give you the idea (and to show that type providers are really nice :-)):
First, I need some references:
#r "System.Xml.Linq.dll"
#r "FSharp.Data.dll"
open FSharp.Data
open FSharp.Net
Next, I wrote the following code to download one sample page.
let data =
Http.Request
( "http://cdn.marketplaceedgeservice.windowsphone.com//v8/ratings/product/4e08377c-1240-4f80-9c35-0bacde2c66b6/reviews",
query=["os", "8.0.0.0"; "cc", "US"; "lang", "en-US"; "hw", "520170499"; "dm", "Test"; "chunksize", "10" ],
headers=["User-Agent", "F#"])
I saved the sample as D:\temp\appstore.xml and then used the XML type provider to get a nice type for parsing the page:
type PageDocument = XmlProvider< #"D:\temp\appstore.xml" >
Then you can download & parse the page like this (this shows how to get the number of reviews and information about the next link):
let parseAsync (locale:string) appId = async {
let country = locale.Substring(3)
// Make the request (asynchronously) using the parameters specified
let! data =
Http.AsyncRequest
( "http://cdn.marketplaceedgeservice.windowsphone.com//v8/ratings/product/"
+ appId + "/reviews",
query=[ "os", "8.0.0.0"; "cc", country; "lang", locale;
"hw", "520170499"; "dm", "Test"; "chunksize", "10" ],
headers=["User-Agent", "F#"])
// Parse the result using the type-provider generated type
let page = PageDocument.Parse(data)
// Now you can type 'page' followed by '.' and explore the results!
// page.GetLinks() returns all links and page.GetEntries() returns
// review entries. Each link also has 'Rel' and 'Href' properties:
let nextLink =
page.GetLinks()
|> Seq.tryFind (fun link -> link.Rel = "next")
|> Option.map (fun link -> link.Href)
let reviewsCount = page.GetEntries().Length
return (reviewsCount, nextLink) }

Answer 2

The general pattern for making code asynchronous is to find the I/O expensive operation (somewhere down in the call tree) and then go "up" from there and make all code that uses it asynchronous too until you reach a point where you need to block.
In your example, the primitive operation is downloading, so you would start by making downloadPage asynchronous:
let downloadPage (uri: System.Uri) = async {
try
use webClient = new System.Net.WebClient()
printfn "%s" (uri.ToString())
// (Headers omitted)
let! source = webClient.AsyncDownloadString(uri)
return { Uri = uri; Source = source; ErrorOccured = false }
with error ->
return { Uri = uri; Source = String.Empty; ErrorOccured = true } }
You need to wrap code in async { ... }, make call to asynchronous version of DownloadString using let! and return the results using return (in both branches).
Then you need to make functions like downloadReview and downloadReviews (again, wrap them in async block, call other asynchronous operations like downloadPage using let! or using return!).
In the end, if you're writing console application you'll need to block, but you can run downloads for different locales in parallel. Assuming downloadReviews is asynchronous:
let locales = [| "en-US"; "en-GB"; |]
let results =
locales
|> Array.map downloadReviews // Build an array of asynchronous computations
|> Async.Parallel // Compose them into a single, parallel computation
|> Async.RunSynchronously // Run the computation and wait
To answer other questions, I think using null in the example above is probably okay (you are calling LINQ which returns it, so there is no easy way to avoid that). It is actually possible to use option type instead, but it is a bit tricky - see this snippet if you're interested.
Also, you could use the Http.AsyncRequest method from F# Data Library which gives you a bit simpler way to construct complex HTTP requests (but I'm one of the contributors to that library, so I'm biased!)

Answer 3

As Tomas said, it would be more "functional" to create an async-based version of DownloadString (or just use his FSharp.Data library to handle it).
You could also combine FSharp.Data with ExtCore to take advantage of the asyncMaybe or asyncChoice workflows in ExtCore. Those workflows provide very easy-to-use error handling on top of the normal async workflow.
Anyway, I spent a few minutes cleaning up your code. It's not much, but it does simplify your code in a few spots:
open System
open System.IO
open System.Xml
open System.Xml.Linq
open Printf
type DownloadPageResult = {
Uri : System.Uri;
ErrorOccured : bool;
Source : string;
}
type ReviewData = {
CurrentPageUri : System.Uri;
NextPageUri : System.Uri option;
NumberOfReviews : uint32;
}
module ReviewUrl =
let baseUri = Uri ("http://cdn.marketplaceedgeservice.windowsphone.com/", UriKind.Absolute)
let getUri country locale (appId : System.Guid) =
let localUri =
let appIdStr = appId.ToString "D"
sprintf "/v8/ratings/product/%s/reviews?os=8.0.0.0&cc=%s&oc=&lang=%s&hw=520170499&dm=Test&chunksize=10" appIdStr country locale
Uri (baseUri, localUri)
let downloadPage (uri : System.Uri) =
try
use webClient = new System.Net.WebClient()
printfn "%s" (uri.ToString())
webClient.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
webClient.Headers.Add("Accept-Encoding", "zip,deflate,sdch")
webClient.Headers.Add("Accept-Language", "en-US,en;q=0.8,fr;q=0.6")
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1482.0 Safari/537.36")
{ Uri = uri; Source = webClient.DownloadString uri; ErrorOccured = false }
with error ->
{ Uri = uri; Source = String.Empty; ErrorOccured = true }
let parse (pageResult : DownloadPageResult) =
if pageResult.ErrorOccured then
{ CurrentPageUri = pageResult.Uri; NextPageUri = None; NumberOfReviews = 0u }
else
use reader = new StringReader (pageResult.Source)
let doc = XDocument.Load reader
let ns = XNamespace.Get "http://www.w3.org/2005/Atom"
let nextUrl =
query {
for link in doc.Descendants(ns + "link") do
where (link.Attribute(XName.Get("rel")).Value = "next")
select link.Value
headOrDefault }
{ CurrentPageUri = pageResult.Uri;
NextPageUri =
if System.String.IsNullOrEmpty nextUrl then None
else Some <| Uri (ReviewUrl.baseUri, nextUrl);
NumberOfReviews =
doc.Descendants (ns + "entry") |> Seq.length |> uint32; }
let downloadReviews (locale : string) =
System.Guid "4e08377c-1240-4f80-9c35-0bacde2c66b6"
|> ReviewUrl.getUri (locale.Substring 3) locale
|> downloadPage
|> parse
[<EntryPoint>]
let main argv =
let locales = [| "en-US"; "en-GB"; |]
let results = locales |> Array.map downloadReviews
printfn "%A" results
0