In mapreduce I have a scope total_count to calculate something after grouping something.
How can I get back the that scope varriable in output?
EDIT:
map = %Q{
function() {
ttotal_commission += this.commission;
tuniq_commission += this.commission; // add first if it not goto reduce
emit({ip: this.ip, campaign_id: this.campaign_id}, {commission: this.commission, abc: this.commission});
}
}
reduce = %Q{
function(key, values) {
tuniq_commission += values[0].commission; // add 1 and substract laster for uniq
var total_commission = 0;
values.forEach(function(value) {
tuniq_commission -= value.commission;
total_commission += value.commission;
});
return {};
}
}
final = %Q{
function(key, reduceVal){
reduceVal.ttotal_commission = ttotal_commission;
reduceVal.tuniq_commission = tuniq_commission;
return reduceVal;
}
}
temp = Transaction.all.limit(45000).map_reduce(map, reduce).scope(ttotal_commission: 0, tuniq_commission: 0).finalize(final).out(inline: true).first
Related
I'm trying to implement my own neat implementation and I can't get myself to understand how speciation works
I tried my best to follow the pesudocode I found in this paper (start of page 13)
but I'm think I'm doing it really wrong but I don't understand the right way to do it, here is my code
the sepciate function that splits the population into species:
function speciate(population, species=[]) {
let newSpecies = [...species];
for(const net of population) {
let placed = false;
for(const s of newSpecies) {
for(const member of s) {
if(sh(net, member)) {
s.push(net);
placed = true;
break;
}
}
if(placed) break;
}
if(!placed) {
newSpecies.push([net]);
}
}
return newSpecies;
}
the repopulation function that generates a new population using the number of offsprings:
function repopulate(popCount, species) {
let globalAvg = 0;
species.forEach(s => {
globalAvg += s.reduce((P, net) => P + net.genome.fitness, 0) / s.length;
});
let newPop = [];
for(const s of species) {
let N = popCount;
let sAvg = s.reduce((P, net) => P + net.genome.fitness, 0) / s.length;
let offspringCount = (sAvg / globalAvg) * N;
for(let i = 0; i < offspringCount; i++) {
let parent1 = wheelSelect(s);
let parent2 = wheelSelect(s);
let child = parent1.genome.crossover(parent2.genome);
child.mutateAddNeuron(0.01);
child.mutateAddConnection(0.01);
child.mutateWeight(0.01);
child.mutateEnabledToggle(0.01);
child.layerNeurons();
let net = new NeuralNetwork();
net.wireUsingGenome(child);
newPop.push(net);
}
}
return newPop;
}
the problem I'm facing is that the population number seems to change every new generation sometimes it goes up and sometimes down, so I'm gussing I'm calculating the offspring count wrong or my speciation isn't working correctly but I can't figure it out
any help is appreciated!
I have a Rails app where I have a search results page, which pulls results based on a user's input from a form.
To simplify the Rails controller method:
def search_results
# [...] ActiveRecord queries to get results
#movies.shuffle!
end
Then in the view, I only want to display the first three results. At the moment, that's something like:
<%#movies[0..2].each do |movie|%>
<div class="row d-flex justify-content-center" data-controller="search-result-card">
<%# Visual of the card %>
</div>
<%end%>
What I want to do is have a refresh button on this page replaces the visible cards with the next three #movies, but without actually reloading the page.
I've drawn up a refresh button and I've got a data-controller on it... but I'm just not sure how I can use this to change the code.
<div class="refresh-button" data-controller="refresh">
<button data-action="click->refresh#next">Refresh</button>
</div>
I'm not sure whether I need to create a partial of the card... or if there's a way to simply change the imbedded Ruby so that <%#movies[0..2].each do |movie|%> becomes <%#movies[3..5].each do |movie|%> and have the view update accordingly.
I don't think that rendering and then hiding all of the results would be feasible, because there could be tens of thousands of them potentially (although I could be wrong and this might be fine to do).
Any suggestions most appreciated!
This sounds like plain old pagination.
If there could be thousands of movies you'd be generation markup/html for thousands of movies. With pagination you'd get get 3 records at a time. Could even use a turbo-frame to replace the next 3.
But if you really want to do what your thinking, you'd have hide all cards except for the first 3, then use stimulus to get the next or last three cards.
A test example
Controller
class TestController < ApplicationController
def index
#movies = Array.new(100) { |e| e = e + 1 }
end
Template using slim
.bg-white
div[data-controller="movies"]
- cnt = 0
button.btn[data-action="click->movies#last_set"] Last
button.btn[data-action="click->movies#next_set"] Next
- #movies.each do |m|
- if cnt < 3
div[data-movies-target="movie" class='block'] = "This is movie #{m}"
- cnt += 1
- else
div[data-movies-target="movie" class='hidden'] = "This is movie #{m}"
Stimulus Controller
import { Controller } from "#hotwired/stimulus"
// Connects to data-controller="movies"
export default class extends Controller {
static targets = ['movie']
connect() {
this.size = this.movieTargets.length
this.idx = 0
// this.idx points to the first current cards (3 for example)
}
next_set() {
let curr = this.idx
this.idx = curr + 3 // set new idx
if (this.idx > this.size) {this.idx = this.size - 3}
for (var i = curr; i <= curr + 2; i++) {
this.movieTargets[i].classList.add("hidden")
}
for (var i = this.idx; i <= this.idx + 2; i++) {
this.movieTargets[i].classList.remove("hidden")
}
// console.log(`idx = ${this.idx}`)
}
last_set() {
let curr = this.idx
this.idx = curr - 3 // set new idx
if (this.idx < 0 ) {this.idx = 0}
for (var i = curr; i <= curr + 2; i++) {
this.movieTargets[i].classList.add("hidden")
}
for (var i = this.idx; i <= this.idx + 2; i++) {
this.movieTargets[i].classList.remove("hidden")
}
// console.log(`idx = ${this.idx}`)
}
}
Just a rough example.
EDIT Not very good at javascript and cleaned up the stimulus controller, added variable number of elements to display and work with pages instead of elements.
import { Controller } from "#hotwired/stimulus"
export default class extends Controller {
static targets = ['perPage','movie','status','nextBtn','lastBtn']
connect() {
this.size = this.movieTargets.length
this.page = 0
this.perPage = Number(this.perPageTarget.innerHTML)
this.pages = Math.floor(this.size/this.perPage)
if ((this.size % this.perPage) > 0) {this.pages += 1}
this.setStatus()
// this.page points to the current set of cards (3 for example)
}
next_set() {
if (this.page == (this.pages - 1)) {return}
this.hide()
this.page += 1
this.show()
this.setStatus()
}
last_set() {
if (this.page == 0) {return}
this.hide()
this.page -= 1
this.show()
this.setStatus()
}
hide(){
let curr = this.page * this.perPage
let last = curr + this.perPage - 1
while(curr <= last){
if (curr >= 0 && curr < this.size) {
this.movieTargets[curr].classList.add("hidden")
}
curr++
}
}
show(){
let curr = this.page * this.perPage
let last = curr + this.perPage - 1
while(curr <= last){
if (curr >= 0 && curr < this.size) {
this.movieTargets[curr].classList.remove("hidden")
}
curr++
}
}
setStatus(){
this.statusTarget.innerHTML = `Page ${this.page + 1} of ${this.pages}`
if (this.page == 0) {
this.lastBtnTarget.classList.add('hidden')
}else{
this.lastBtnTarget.classList.remove('hidden')
}
if (this.page == this.pages) {
this.nextBtnTarget.classList.add('hidden')
}else{
this.nextBtnTarget.classList.remove('hidden')
}
}
}
Basically I have two google sheets that look something like this:
a table where people can put in their email and select what kind of foo they are using
email
foo
example#email.com
This Foo
and then another table with information about the foo
foo name
foo type
foo boolean1
foo boolean2
This Foo
String
True
True
That Foo
Number
False
True
Other Foo
String
False
False
In a Separate Sheet I'd like to have a dashboard-like view of things wherein I would have counts of various things like number of people, how many of each type of Foo, etc
Where I'm having trouble is figuring out how to pull things like "Number of people who have selected String foos" and such
like, basically i want the google-query equivalent to (in sql)
SELECT COUNT(p.*) FROM people p JOIN info i on p.foo = i.foo_name GROUP BY i.foo_type WHERE i.foo_type = 'String'
What I would be looking for is a table that looks like this:
Data
Count
Active Roster
4
String
3
Number
1
I have also seen many solutions that have complicated formulas using VLOOKUP, INDEX, MATCH, etc.
I decided to write a user function to combine tables, or as I refer to it, de-normalize the database. I wrote the function DENORMALIZE() to support INNER, LEFT, RIGHT and FULL joins. By nesting function calls one can join unlimited tables in theory.
DENORMALIZE(range1, range2, primaryKey, foreignKey, [joinType])
Parameters:
range1, the main table as a named range, a1Notation or an array
range2, the related table as a named range, a1Notation or an array
primaryKey, the unique identifier for the main table, columns start with "1"
foreignKey, the key in the related table to join to the main table, columns start with "1"
joinType, type of join, "Inner", "Left", "Right", "Full", optional and defaults to "Inner", case insensitive
Returns: results as a two dimensional array
Result Set Example:
=QUERY(denormalize("Employees","Orders",1,3), "SELECT * WHERE Col2 = 'Davolio' AND Col8=2", FALSE)
EmpID
LastName
FirstName
OrderID
CustomerID
EmpID
OrderDate
ShipperID
1
Davolio
Nancy
10285
63
1
8/20/1996
2
1
Davolio
Nancy
10292
81
1
8/28/1996
2
1
Davolio
Nancy
10304
80
1
9/12/1996
2
Other Examples:
=denormalize("Employees","Orders",1,3)
=denormalize("Employees","Orders",1,3,"full")
=QUERY(denormalize("Employees","Orders",1,3,"left"), "SELECT * ", FALSE)
=QUERY(denormalize("Employees","Orders",1,3), "SELECT * WHERE Col2 = 'Davolio'", FALSE)
=QUERY(denormalize("Employees","Orders",1,3), "SELECT * WHERE Col2 = 'Davolio' AND Col8=2", FALSE)
=denormalize("Orders","OrderDetails",1,2)
// multiple joins
=denormalize("Employees",denormalize("Orders","OrderDetails",1,2),1,3)
=QUERY(denormalize("Employees",denormalize("Orders","OrderDetails",1,2),1,3), "SELECT *", FALSE)
=denormalize(denormalize("Employees","Orders",1,3),"OrderDetails",1,2)
=QUERY(denormalize("Employees",denormalize("Orders","OrderDetails",1,2),1,3), "SELECT *", FALSE)
=QUERY(denormalize(denormalize("Employees","Orders",1,3),"OrderDetails",4,2), "SELECT *", FALSE)
function denormalize(range1, range2, primaryKey, foreignKey, joinType) {
var i = 0;
var j = 0;
var index = -1;
var lFound = false;
var aDenorm = [];
var hashtable = [];
var aRange1 = "";
var aRange2 = "";
joinType = DefaultTo(joinType, "INNER").toUpperCase();
// the 6 lines below are used for debugging
//range1 = "Employees";
//range1 = "Employees!A2:C12";
//range2 = "Orders";
//primaryKey = 1;
//foreignKey = 3;
//joinType = "LEFT";
// Sheets starts numbering columns starting with "1", arrays are zero-based
primaryKey -= 1;
foreignKey -= 1;
// check if range is not an array
if (typeof range1 !== 'object') {
// Determine if range is a1Notation and load data into an array
if (range1.indexOf(":") !== -1) {
aRange1 = ss.getRange(range1).getValues();
} else {
aRange1 = ss.getRangeByName(range1).getValues();
}
} else {
aRange1 = range1;
}
if (typeof range2 !== 'object') {
if (range2.indexOf(":") !== -1) {
aRange2 = ss.getRange(range2).getValues();
} else {
aRange2 = ss.getRangeByName(range2).getValues();
}
} else {
aRange2 = range2;
}
// make similar structured temp arrays with NULL elements
var tArray1 = MakeArray(aRange1[0].length);
var tArray2 = MakeArray(aRange2[0].length);
var lenRange1 = aRange1.length;
var lenRange2 = aRange2.length;
hashtable = getHT(aRange1, lenRange1, primaryKey);
for(i = 0; i < lenRange2; i++) {
index = hashtable.indexOf(aRange2[i][foreignKey]);
if (index !== -1) {
aDenorm.push(aRange1[index].concat(aRange2[i]));
}
}
// add left and full no matches
if (joinType == "LEFT" || joinType == "FULL") {
for(i = 0; i < lenRange1; i++) {
//index = aDenorm.indexOf(aRange1[i][primaryKey]);
index = aScan(aDenorm, aRange1[i][primaryKey], primaryKey)
if (index == -1) {
aDenorm.push(aRange1[i].concat(tArray2));
}
}
}
// add right and full no matches
if (joinType == "RIGHT" || joinType == "FULL") {
for(i = 0; i < lenRange2; i++) {
index = aScan(aDenorm, aRange2[i][foreignKey], primaryKey)
if (index == -1) {
aDenorm.push(tArray1.concat(aRange2[i]));
}
}
}
return aDenorm;
}
function getHT(aRange, lenRange, key){
var aHashtable = [];
var i = 0;
for (i=0; i < lenRange; i++ ) {
//aHashtable.push([aRange[i][key], i]);
aHashtable.push(aRange[i][key]);
}
return aHashtable;
}
function MakeArray(length) {
var i = 0;
var retArray = [];
for (i=0; i < length; i++) {
retArray.push("");
}
return retArray;
}
function DefaultTo(valueToCheck, valueToDefault) {
return typeof valueToCheck === "undefined" ? valueToDefault : valueToCheck;
}
// Search a multi-dimensional array for a value
function aScan(aValues, searchStr, searchCol) {
var retval = -1;
var i = 0;
var aLen = aValues.length;
for (i = 0; i < aLen; i++) {
if (aValues[i][searchCol] == searchStr) {
retval = i;
break;
}
}
return retval;
}
You can make a copy of the google sheet with data and examples here:
https://docs.google.com/spreadsheets/d/1vziuF8gQcsOxTLEtlcU2cgTAYL1eIaaMTAoIrAS7mnE/edit?usp=sharing
How to sort the items within each stacking column? Asc or desc.
Each series added to a chart is drawn on the chart in the order it was received. To change the order of the chart series you will need to change which series is the first in your list of series items.
That being said - what I think you want to do is to, independently of the series order, sort on each stack by value. I do not think this is possible in HighCharts.
You can only set global index of serie, but you cannot position each single "stack".
http://api.highcharts.com/highcharts#series.index
You may use the script below to sort the Stacked Chart Bars by category name.
var sortData = function(chartSource) {
var series = chartSource.series;
var axis = chartSource.xAxis[0];
var categories = [];
if($.isArray(series)) {
var ser =
$.grep(series, function(ser, seriesIndex)
{
return ser.visible;
})[0];
$.each(ser.data,
function(dataIndex, datum)
{
console.log(datum.category + ':' + datum.stackTotal);
var obj = {
name: datum.category,
index: dataIndex,
stackTotal: datum.stackTotal
}
categories.push(obj);
}
);
}
categories.sort(function(a, b) {
var aName = a.name.toLowerCase();
var bName = b.name.toLowerCase();
var aTotal = a.stackTotal;
var bTotal = b.stackTotal;
//if(aTotal === bTotal) {
return ((aName < bName) ? -1 : ((aName > bName) ? 1 : 0));
//} else {
// return ((aTotal > bTotal) ? -1 : ((aTotal < bTotal) ? 1 : 0));
//}
});
var mappedIndex = $.map(categories, function(category, index) {
return category.index;
});
categories = $.map(categories, function(category, index) {
return category.name;
});
console.log(categories);
console.log(mappedIndex);
axis.setCategories(categories);
var newDataArray = [];
$.each(series, function(seriesIndex, ser) {
newDataArray = [];
var data = $.map(mappedIndex, function(mappedIndex2, origIndex) {
var ydata = ser.data[mappedIndex2];
if(ydata.y!=null){
var y = ydata.y
newDataArray.push(y);
return y;
}
else
{
newDataArray.push(null);
return null;
}
});
ser.setData(newDataArray);
});
};
How can I compute a MD5 or SHA1 hash of text in a specific cell and set it to another cell in Google Spreadsheet?
Is there a formula like =ComputeMD5(A1) or =ComputeSHA1(A1)?
Or is it possible to write custom formula for this? How?
Open Tools > Script Editor then paste the following code:
function MD5 (input) {
var rawHash = Utilities.computeDigest(Utilities.DigestAlgorithm.MD5, input);
var txtHash = '';
for (i = 0; i < rawHash.length; i++) {
var hashVal = rawHash[i];
if (hashVal < 0) {
hashVal += 256;
}
if (hashVal.toString(16).length == 1) {
txtHash += '0';
}
txtHash += hashVal.toString(16);
}
return txtHash;
}
Save the script after that and then use the MD5() function in your spreadsheet while referencing a cell.
This script is based on Utilities.computeDigest() function.
Thanks to gabhubert for the code.
This is the SHA1 version of that code (very simple change)
function GetSHA1(input) {
var rawHash = Utilities.computeDigest(Utilities.DigestAlgorithm.SHA_1, input);
var txtHash = '';
for (j = 0; j <rawHash.length; j++) {
var hashVal = rawHash[j];
if (hashVal < 0)
hashVal += 256;
if (hashVal.toString(16).length == 1)
txtHash += "0";
txtHash += hashVal.toString(16);
}
return txtHash;
}
Ok, got it,
Need to create custom function as explained in
http://code.google.com/googleapps/appsscript/articles/custom_function.html
And then use the apis as explained in
http://code.google.com/googleapps/appsscript/service_utilities.html
I need to handtype the complete function name so that I can see the result in the cell.
Following is the sample of the code that gave base 64 encoded hash of the text
function getBase64EncodedMD5(text)
{
return Utilities.base64Encode( Utilities.computeDigest(Utilities.DigestAlgorithm.MD5, text));
}
The difference between this solution and the others is:
It fixes an issue some of the above solution have with offsetting the output of Utilities.computeDigest (it offsets by 128 instead of 256)
It fixes an issue that causes some other solutions to produce the same hash for different inputs by calling JSON.stringify() on input before passing it to Utilities.computeDigest()
function MD5(input) {
var result = "";
var byteArray = Utilities.computeDigest(Utilities.DigestAlgorithm.MD5, JSON.stringify(input));
for (i=0; i < byteArray.length; i++) {
result += (byteArray[i] + 128).toString(16) + "-";
}
result = result.substring(result, result.length - 1); // remove trailing dash
return result;
}
to get hashes for a range of cells, add this next to gabhubert's function:
function RangeGetMD5Hash(input) {
if (input.map) { // Test whether input is an array.
return input.map(GetMD5Hash); // Recurse over array if so.
} else {
return GetMD5Hash(input)
}
}
and use it in cell this way:
=RangeGetMD5Hash(A5:X25)
It returns range of same dimensions as source one, values will spread down and right from cell with formulae.
It's universal single-value-function to range-func conversion method (ref), and it's way faster than separate formuleas for each cell; in this form, it also works for single cell, so maybe it's worth to rewrite source function this way.
Based on #gabhubert but using array operations to get the hexadecimal representation
function sha(str){
return Utilities
.computeDigest(Utilities.DigestAlgorithm.SHA_1, str) // string to digested array of integers
.map(function(val) {return val<0? val+256 : val}) // correct the offset
.map(function(val) {return ("00" + val.toString(16)).slice(-2)}) // add padding and enconde
.join(''); // join in a single string
}
Using #gabhubert answer, you could do this, if you want to get the results from a whole row. From the script editor.
function GetMD5Hash(value) {
var rawHash = Utilities.computeDigest(Utilities.DigestAlgorithm.MD5, value);
var txtHash = '';
for (j = 0; j <rawHash.length; j++) {
var hashVal = rawHash[j];
if (hashVal < 0)
hashVal += 256;
if (hashVal.toString(16).length == 1)
txtHash += "0";
txtHash += hashVal.toString(16);
}
return txtHash;
}
function straightToText() {
var ss = SpreadsheetApp.getActiveSpreadsheet().getSheets();
var r = 1;
var n_rows = 9999;
var n_cols = 1;
var column = 1;
var sheet = ss[0].getRange(r, column, n_rows, ncols).getValues(); // get first sheet, a1:a9999
var results = [];
for (var i = 0; i < sheet.length; i++) {
var hashmd5= GetMD5Hash(sheet[i][0]);
results.push(hashmd5);
}
var dest_col = 3;
for (var j = 0; j < results.length; j++) {
var row = j+1;
ss[0].getRange(row, dest_col).setValue(results[j]); // write output to c1:c9999 as text
}
}
And then, from the Run menu, just run the function straightToText() so you can get your result, and elude the too many calls to a function error.
I was looking for an option that would provide a shorter result. What do you think about this? It only returns 4 characters. The unfortunate part is that it uses i's and o's which can be confused for L's and 0's respectively; with the right font and in caps it wouldn't matter much.
function getShortMD5Hash(input) {
var rawHash = Utilities.computeDigest(Utilities.DigestAlgorithm.MD5, input);
var txtHash = '';
for (j = 0; j < 16; j += 8) {
hashVal = (rawHash[j] + rawHash[j+1] + rawHash[j+2] + rawHash[j+3]) ^ (rawHash[j+4] + rawHash[j+5] + rawHash[j+6] + rawHash[j+7])
if (hashVal < 0)
hashVal += 1024;
if (hashVal.toString(36).length == 1)
txtHash += "0";
txtHash += hashVal.toString(36);
}
return txtHash.toUpperCase();
}
I needed to get a hash across a range of cells, so I run it like this:
function RangeSHA256(input)
{
return Array.isArray(input) ?
input.map(row => row.map(cell => SHA256(cell))) :
SHA256(input);
}