I'm trying to get the size of a key-value or just a key or just a value in redis.
Using debug object key command returns the serialized size of a key-value if it were to be written to disk and not the actual amount of bytes it is using in memory.
This can be confirmed by checking the source code (based from this Redis: Show database size/size for keys)
https://github.com/antirez/redis/blob/4082c38a60eedd524c78ef48c1b241105f4ddc50/src/debug.c#L337-L343
https://github.com/antirez/redis/blob/4082c38a60eedd524c78ef48c1b241105f4ddc50/src/rdb.c#L663-L671
looking at the source code:
/* Save a string object as [len][data] on disk. If the object is a string
* representation of an integer value we try to save it in a special form */
ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len) {
int enclen;
ssize_t n, nwritten = 0;
/* Try integer encoding */
if (len <= 11) {
unsigned char buf[5];
if ((enclen = rdbTryIntegerEncoding((char*)s,len,buf)) > 0) {
if (rdbWriteRaw(rdb,buf,enclen) == -1) return -1;
return enclen;
}
}
/* Try LZF compression - under 20 bytes it's unable to compress even
* aaaaaaaaaaaaaaaaaa so skip it */
if (server.rdb_compression && len > 20) {
n = rdbSaveLzfStringObject(rdb,s,len);
if (n == -1) return -1;
if (n > 0) return n;
/* Return value of 0 means data can't be compressed, save the old way */
}
/* Store verbatim */
if ((n = rdbSaveLen(rdb,len)) == -1) return -1;
nwritten += n;
if (len > 0) {
if (rdbWriteRaw(rdb,s,len) == -1) return -1;
nwritten += len;
}
return nwritten;
}
And confirm through redis-cli:
127.0.0.1:6379> set a aaaaaaaaaaaaaaaaaaa
OK
127.0.0.1:6379> debug object a
Value at:0x7f985822f168 refcount:1 encoding:embstr serializedlength:20 lru:11611136 lru_seconds_idle:2
127.0.0.1:6379> set a aaaaaaaaaaaaaaaaaaaaaaaaaaaaa
OK
127.0.0.1:6379> debug object a
Value at:0x7f985827c428 refcount:1 encoding:embstr serializedlength:12 lru:11611147 lru_seconds_idle:1
All the different CLI tools are reporting the serialized size of the object and not the memory size which is the interesting and important one.
As of Redis v4, the MEMORY USAGE command does a much better job at guessing the footprint of a key and its value.
Related
Hi please find my code below, I am trying to write to SRAM. please help
my code below reads the output from a cell but i can't write to that cell
CS: pin 12
MOSI: pin 8
MISO: pin 10
SCK: pin 9
*/
#include <SPI.h>
//SRAM opcodes
#define RDSRAM 5 //00000101
#define WRSRAM 1 //00000001
#define READ 3 //00000011
#define WRITE 2 //00000010
int *ptr;
int CS = 12;
int CSS = 8;
char buf [90];
int response_pair;
int entryval;
int codeAddr = 545;
char s [90];
//char value = *(char*)0x5C;
uint8_t Spi23K640Rd8(uint32_t address){
uint8_t read_byte;
digitalWrite(CS,LOW);
SPI.transfer(READ);
//SPI.transfer((uint8_t)(address >> 16) & 0xff);
SPI.transfer((uint8_t)(address >> 8) & 0xff);
SPI.transfer((uint8_t)address);
read_byte = SPI.transfer(0x00);
digitalWrite(CS,HIGH);
return read_byte;
}
void Spi23K640Wr8(uint32_t address, uint8_t data_byte)
{
SPI.transfer(WRITE);
SPI.transfer((uint8_t)(address >> 16) & 0xff);
SPI.transfer((uint8_t)(address >> 8) & 0xff);
SPI.transfer((uint8_t)address);
SPI.transfer(data_byte);
}
void setup(void) {
// char *ptr;
// char myvar[1] = {545};
uint64_t i;
uint8_t value;
ptr=&codeAddr;
/* all pins on the Port B set to output-low */
pinMode(CSS, OUTPUT);
digitalWrite(CSS, HIGH);
pinMode(CS, OUTPUT);
Serial.begin(9600);
delay(2500);
SPI.begin();
for (i=0; i<=8192; i++) { // Do all memory locations, 64 Kbit SRAM = 65536 / 8 = 8192
Spi23K640Wr8(i, (uint8_t)i);
value = Spi23K640Rd8(i);
Serial.print((uint64_t)value, DEC);
if ( !(i % 32) && !(i==0) ) { // Every 32, do a new line and don't do the first item either
Serial.println(value);
} else
{ // Other wise, print a comma
Serial.print(",");
}
}
while (!Serial) ;
int response_pair = Spi23K640Rd8 (codeAddr);
Serial.println ("Enter Challenge");
//Spi23K640Wr8();
delay(500);
}
void loop() {
//while (!Serial) ;
int response_pair = Spi23K640Rd8 (codeAddr);
if (Serial.available ()) {
int n = Serial.readBytesUntil ('\n', buf, sizeof (buf)-1); //.toInt(); //save read value method to n
buf [n] = '\0';
sscanf (buf, "%o", &entryval); //check values
sprintf (s, " buf %s, response_pair %o entryval %o", buf, response_pair, entryval); //point the values from the pointer
if(entryval == response_pair )
{
Serial.println ("RESPONSE PAIR MATCHES ");
Serial.println ("loading address......");
Serial.print ("CRP address output = ");
Serial.println (Spi23K640Rd8(codeAddr), DEC); //prints out specific address
Serial.println ("Authenticate Chip");
Serial.println (s);
//delay (500);
}
else if (response_pair != entryval)
{
Serial.println ("INTRUDER ALERT!!!Wrong challenge");
Serial.println (s); //print the values in different types
delay (n);
}
// return;
// while (!Serial) ;
Serial.println();
Serial.println ("Enter Another Challenge"); //start the process again
//Serial.println (s); //print the values in different types
}
// put your main code here, to run repeatedly:
//Serial.println("Hello LoRa");
//delay(50);
//ptr++;
}
i was able to read the power up state but haven't had any luck writing to the SRAM cells
any suggestions will be appreciated. i am on a tight schedule.
disregard beloww
We have established that In order to evaluate the properties of the SRAM as a PUF, we perform a number of specifically selected tests to investigate the behaviour of the start-up values of the SRAM memory
• The technique can be viewed as an attempt to read multiple cells in a column at the same time, creating contention that is resolved according to process variation
• An authentication challenge is issued to the array of SRAM cells by activating two or more wordlines concurrently
• The response is simply the value that the SRAM produces from a read operation when the challenge condition is applied
• The number of challenges that can be applied the array of SRAM cells grows exponentially with the number of SRAM rows and these challenges can be applied at any time without power cycling
• providing an array of different responses on different chips ; these challenges are SRAM cells arranged in rows and columns where SRAM cells in each column and array share a worldlines
• SRAM cells in each column in the array share common is a graph illustrating the number of unbiased bit lines
The CS line is not asserted during the write operation, and the SRAM uses 16 and not 24-bit addresses. You can try changing your read and write functions to something like this:
uint8_t Spi23K640Rd8(uint16_t address) { // <-- change from uint32_t to uint16_t
uint8_t read_byte;
digitalWrite(CS, LOW); // That's good
SPI.transfer(READ); // Read # 16-bit address, that's good
SPI.transfer((uint8_t)(address >> 8) & 0xff);
SPI.transfer((uint8_t)address);
read_byte = SPI.transfer(0x00);
digitalWrite(CS, HIGH);
return read_byte;
}
void Spi23K640Wr8(uint16_t address, uint8_t data_byte) { // <-- change from uint32_t to uint16_t
digitalWrite(CS, LOW); // Was missing.
SPI.transfer(WRITE); // write #16-bit address
// SPI.transfer((uint8_t)(address >> 16) & 0xff); // <- BUG!!! this byte is not expected!
SPI.transfer((uint8_t)(address >> 8) & 0xff);
SPI.transfer((uint8_t)address);
SPI.transfer(data_byte);
digitalWrite(CS, HIGH); // clear CS.
}
Note: You should also consider renaming these functions to make your code easier to read.
How about replacing
uint8_t Spi23K640Rd8(uint16_t address);
void Spi23K640Wr8(uint16_t address, uint8_t data_byte);
With
uint8_t SRAM_23K640_ReadByte(uint16_t address);
void SRAM_23K640_WriteByte(uint16_t address, uint8_t data_byte);
Or whatever you see fit. Keep in mind that our eyes and brain have a much easier time reading shorter, pronounceable words. When the brain is too busy reading long mumbo jumbo, thinking about other things, like what the code does, becomes more difficult.
What I want is extracting a value from vector using a variable scalar index.
Like _mm_extract_epi8 / _mm256_extract_epi8 but with non-immediate input.
(There are some results in the vector, the one with the given index is found out to be the true result, the rest are discarded)
Especially, if index is in a GPR, the easiest way is probably to store val to memory and then movzx it into another GPR. Sample implementation using C:
uint8_t extract_epu8var(__m256i val, int index) {
union {
__m256i m256;
uint8_t array[32];
} tmp;
tmp.m256 = val;
return tmp.array[index];
}
Godbolt translation (note that a lot of overhead happens for stack alignment -- if you don't have an aligned temporary storage area, you could just vmovdqu instead of vmovdqa): https://godbolt.org/z/Gj6Eadq9r
So far the best option seem to be using _mm_shuffle_epi8 for SSE
uint8_t extract_epu8var(__m128i val, int index) {
return (uint8_t)_mm_cvtsi128_si32(
_mm_shuffle_epi8(val, _mm_cvtsi32_si128(index)));
}
Unfortunately this does not scale well for AVX. vpshufb does not shuffle across lanes. There is a cross lane shuffle _mm256_permutevar8x32_epi32, but the resulting stuff seem to be complicated:
uint8_t extract_epu8var(__m256i val, int index) {
int index_low = index & 0x3;
int index_high = (index >> 2);
return (uint8_t)(_mm256_cvtsi256_si32(_mm256_permutevar8x32_epi32(
val, _mm256_zextsi128_si256(_mm_cvtsi32_si128(index_high))))
>> (index_low << 3));
}
This is with reference to this answer: https://stackoverflow.com/a/5694803/762747
So the BSON Object ID consists of:
[4 bytes seconds since epoch, 3 bytes machine hash, 2 bytes process ID, 3 bytes counter]
What logic does mongoid/moped (http://mongoid.org/) use to generate the 3 bytes counter - do they increment or generate a random number?
If effect, can we be almost certain that the BSON ObjectId generated by mongoid will be unique (and avoid point 2 in the referenced answer)?
We can be almost certain that the ObjectId will be unique. :)
When not using the native extensions (which are now included in the bson gem apparently which is used by mongoid/moped), it's using a counter.
def next(time = nil)
#mutex.lock
begin
count = #counter = (#counter + 1) % 0xFFFFFF
ensure
#mutex.unlock rescue nil
end
generate(time || ::Time.new.to_i, count)
end
It's using a counter as you can see in the Generator class.
When using the native C code for generating an ObjectId, it too is using a counter:
static VALUE rb_object_id_generator_next(int argc, VALUE* time, VALUE self)
{
char bytes[12];
unsigned long t;
unsigned short pid = htons(getpid());
if (argc == 0 || (argc == 1 && *time == Qnil)) {
t = rb_current_time_milliseconds();
}
else {
t = htonl(NUM2UINT(rb_funcall(*time, rb_intern("to_i"), 0)));
}
memcpy(&bytes, &t, 4);
memcpy(&bytes[4], rb_bson_machine_id, 3);
memcpy(&bytes[7], &pid, 2);
memcpy(&bytes[9], (unsigned char*) &rb_bson_object_id_counter, 3);
rb_bson_object_id_counter++;
return rb_str_new(bytes, 12);
}
Addendum (thanks to #ChrisHeald for the tip)
Assuming the Ruby implementation being used is using a GIL (Global Interpreter Lock) and using the C code implementation above, the C code would also be safe when it increments the counter, rb_bson_object_id_counter, as there would be an outer lock on the call to this code.
I have a character buffer that I would like to compress in place. Right now I have it set up so there are two buffers and zlib's deflate reads from the input buffer and writes to the output buffer. Then I have to change the input buffer pointer to point to the output buffer and free the old input buffer. This seems like an unnecessary amount of allocation. Since zlib is compressing, the next_out pointer should always lag behind the next_in pointer. Anyway, I can't find enough documentation to verify this and was hoping someone had some experience with this. Thanks for your time!
It can be done, with some care. The routine below does it. Not all data is compressible, so you have to handle the case where the output data catches up with the input data. It takes a lot of incompressible data, but it can happen (see comments in code), in which case you have to allocate a buffer to temporarily hold the remaining input.
/* Compress buf[0..len-1] in place into buf[0..*max-1]. *max must be greater
than or equal to len. Return Z_OK on success, Z_BUF_ERROR if *max is not
enough output space, Z_MEM_ERROR if there is not enough memory, or
Z_STREAM_ERROR if *strm is corrupted (e.g. if it wasn't initialized or if it
was inadvertently written over). If Z_OK is returned, *max is set to the
actual size of the output. If Z_BUF_ERROR is returned, then *max is
unchanged and buf[] is filled with *max bytes of uncompressed data (which is
not all of it, but as much as would fit).
Incompressible data will require more output space than len, so max should
be sufficiently greater than len to handle that case in order to avoid a
Z_BUF_ERROR. To assure that there is enough output space, max should be
greater than or equal to the result of deflateBound(strm, len).
strm is a deflate stream structure that has already been successfully
initialized by deflateInit() or deflateInit2(). That structure can be
reused across multiple calls to deflate_inplace(). This avoids unnecessary
memory allocations and deallocations from the repeated use of deflateInit()
and deflateEnd(). */
int deflate_inplace(z_stream *strm, unsigned char *buf, unsigned len,
unsigned *max)
{
int ret; /* return code from deflate functions */
unsigned have; /* number of bytes in temp[] */
unsigned char *hold; /* allocated buffer to hold input data */
unsigned char temp[11]; /* must be large enough to hold zlib or gzip
header (if any) and one more byte -- 11
works for the worst case here, but if gzip
encoding is used and a deflateSetHeader()
call is inserted in this code after the
deflateReset(), then the 11 needs to be
increased to accomodate the resulting gzip
header size plus one */
/* initialize deflate stream and point to the input data */
ret = deflateReset(strm);
if (ret != Z_OK)
return ret;
strm->next_in = buf;
strm->avail_in = len;
/* kick start the process with a temporary output buffer -- this allows
deflate to consume a large chunk of input data in order to make room for
output data there */
if (*max < len)
*max = len;
strm->next_out = temp;
strm->avail_out = sizeof(temp) > *max ? *max : sizeof(temp);
ret = deflate(strm, Z_FINISH);
if (ret == Z_STREAM_ERROR)
return ret;
/* if we can, copy the temporary output data to the consumed portion of the
input buffer, and then continue to write up to the start of the consumed
input for as long as possible */
have = strm->next_out - temp;
if (have <= (strm->avail_in ? len - strm->avail_in : *max)) {
memcpy(buf, temp, have);
strm->next_out = buf + have;
have = 0;
while (ret == Z_OK) {
strm->avail_out = strm->avail_in ? strm->next_in - strm->next_out :
(buf + *max) - strm->next_out;
ret = deflate(strm, Z_FINISH);
}
if (ret != Z_BUF_ERROR || strm->avail_in == 0) {
*max = strm->next_out - buf;
return ret == Z_STREAM_END ? Z_OK : ret;
}
}
/* the output caught up with the input due to insufficiently compressible
data -- copy the remaining input data into an allocated buffer and
complete the compression from there to the now empty input buffer (this
will only occur for long incompressible streams, more than ~20 MB for
the default deflate memLevel of 8, or when *max is too small and less
than the length of the header plus one byte) */
hold = strm->zalloc(strm->opaque, strm->avail_in, 1);
if (hold == Z_NULL)
return Z_MEM_ERROR;
memcpy(hold, strm->next_in, strm->avail_in);
strm->next_in = hold;
if (have) {
memcpy(buf, temp, have);
strm->next_out = buf + have;
}
strm->avail_out = (buf + *max) - strm->next_out;
ret = deflate(strm, Z_FINISH);
strm->zfree(strm->opaque, hold);
*max = strm->next_out - buf;
return ret == Z_OK ? Z_BUF_ERROR : (ret == Z_STREAM_END ? Z_OK : ret);
}
I am writing code to ingest the IOR file generated by the team responsible for the server and use it to bind my client to their object. Sounds easy, right?
For some reason a bit beyond my grasp (having to do with firewalls, DMZs, etc.), the value for the server inside the IOR file is not something we can use. We have to modify it. However, the IOR string is encoded.
What does Visibroker provide that will let me decode the IOR string, change one or more values, then re-encode it and continue on as normal?
I've already looked into IORInterceptors and URL Naming but I don't think either will do the trick.
Thanks in advance!
When you feel like you need to hack an IOR, resist the urge to do so by writing code and whatnot to mangle it to your liking. IORs are meant to be created and dictated by the server that contains the referenced objects, so the moment you start mucking around in there, you're kinda "voiding your warranty".
Instead, spend your time finding the right way to make the IOR usable in your environment by having the server use an alternative hostname when it generates them. Most ORBs offer such a feature. I don't know Visibroker's particular configuration options at all, but a quick Google search revealed this page that shows a promising value:
vbroker.se.iiop_ts.host
Specifies the host name used by this server engine.
The default value, null, means use the host name from the system.
Hope that helps.
Long time ago I wrote IorParser for GNU Classpath, the code is available. It is a normal parser written being aware about the format, should not "void a warranty" I think. IOR contains multiple tagged profiles that are encapsulated very much like XML so we could parse/modify profiles that we need and understand and leave the rest untouched.
The profile we need to parse is TAG_INTERNET_IOP. It contains version number, host, port and object key. Code that reads and writes this profile can be found in gnu.IOR class. I am sorry this is part of the system library and not a nice piece of code to copy paste here but it should not be very difficult to rip it out with a couple of dependent classes.
This question has been repeatedly asked as CORBA :: Get the client ORB address and port with use of IIOP
Use the FixIOR tool (binary) from jacORB to patch the address and port of an IOR. Download the binary (unzip it) and run:
fixior <new-address> <new-port> <ior-file>
The tool will override the content of the IOR file with the 'patched' IOR
You can use IOR Parser to check the resulting IOR and compare it to your original IOR
Use this function to change the IOR. pass stringified IOR as first argument.
void hackIOR(const char* str, char* newIOR )
{
size_t s = (str ? strlen(str) : 0);
char temp[1000];
strcpy(newIOR,"IOR:");
const char *p = str;
s = (s-4)/2; // how many octets are there in the string
p += 4;
int i;
for (i=0; i<(int)s; i++) {
int j = i*2;
char v=0;
if (p[j] >= '0' && p[j] <= '9') {
v = ((p[j] - '0') << 4);
}
else if (p[j] >= 'a' && p[j] <= 'f') {
v = ((p[j] - 'a' + 10) << 4);
}
else if (p[j] >= 'A' && p[j] <= 'F') {
v = ((p[j] - 'A' + 10) << 4);
}
else
cout <<"invalid octet"<<endl;
if (p[j+1] >= '0' && p[j+1] <= '9') {
v += (p[j+1] - '0');
}
else if (p[j+1] >= 'a' && p[j+1] <= 'f') {
v += (p[j+1] - 'a' + 10);
}
else if (p[j+1] >= 'A' && p[j+1] <= 'F') {
v += (p[j+1] - 'A' + 10);
}
else
cout <<"invalid octet"<<endl;
temp[i]=v;
}
temp[i] = 0;
// Now temp has decoded IOR string. print it.
// Replace the object ID in temp.
// Encoded it back, with following code.
int temp1,temp2;
int l,k;
for(k = 0, l = 4 ; k < s ; k++)
{
temp1=temp2=temp[k];
temp1 &= 0x0F;
temp2 = temp2 & 0xF0;
temp2 = temp2 >> 4;
if(temp2 >=0 && temp2 <=9)
{
newIOR[l++] = temp2+'0';
}
else if(temp2 >=10 && temp2 <=15)
{
newIOR[l++] = temp2+'A'-10;
}
if(temp1 >=0 && temp1 <=9)
{
newIOR[l++] = temp1+'0';
}
else if(temp1 >=10 && temp1 <=15)
{
newIOR[l++] = temp1+'A'-10;
}
}
newIOR[l] = 0;
//new IOR is present in new variable newIOR.
}
Hope this works for you.