confusing intermittent failure with pycrypto - pycrypto

I've been playing with python's crypto library, and I built a simple threading server to encrypt and decrypt. The problem I'm having is that about 1 out of 3 decryptions comes back incorrectly. Here's the code:
class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
global KEY
request_text = ''
while request_text.rfind("\n") == -1:
sock_data='';
recv_size=8192
sock_data=self.request.recv(recv_size)
if sock_data == '':
print "hangup"
break
request_text = request_text + sock_data
args = json.loads(request_text)
print request_text
print "\n"
if args['command'] == 'encrypt':
iv = Random.new().read(AES.block_size)
cipher = AES.new(KEY, AES.MODE_CFB, iv)
crypted_message = iv + b'|' + cipher.encrypt(unquote_plus(args['message']))
response = {'encrypted_message': binascii.hexlify(crypted_message)}
if args['command'] == 'decrypt':
unhexed = binascii.unhexlify(args['message'])
components = unhexed.split('|')
iv = components[0]
cipher = AES.new(KEY, AES.MODE_CFB, iv)
decrypted_message = cipher.decrypt(components[1])
response = {'decrypted_message': decrypted_message}
self.request.sendall(json.dumps(response) + "\n")
Often, I get this error from python:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/SocketServer.py", line 582, in process_request_thread
self.finish_request(request, client_address)
File "/usr/local/lib/python2.7/SocketServer.py", line 323, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/local/lib/python2.7/SocketServer.py", line 639, in __init__
self.handle()
File "cryptoserver.py", line 40, in handle
cipher = AES.new(KEY, AES.MODE_CFB, iv)
File "build/bdist.macosx-10.4-x86_64/egg/Crypto/Cipher/AES.py", line 95, in new
return AESCipher(key, *args, **kwargs)
File "build/bdist.macosx-10.4-x86_64/egg/Crypto/Cipher/AES.py", line 59, in __init__
blockalgo.BlockAlgo.__init__(self, _AES, key, *args, **kwargs)
File "build/bdist.macosx-10.4-x86_64/egg/Crypto/Cipher/blockalgo.py", line 141, in __init__
self._cipher = factory.new(key, *args, **kwargs)
ValueError: IV must be 16 bytes long
----------------------------------------
but just as often, I get no error, but the decryption doesn't work correctly. I'm using this php to test it:
<?php
include_once("config.php");
function encrypt($text) {
$package = array("command" => "encrypt",
"message" => base64_encode($text));
$package_json = json_encode($package);
$serverSays = transmit($package_json);
$serverSaysArray = json_decode($serverSays);
return $serverSaysArray->encrypted_message;
}
function decrypt($text) {
$package = array("command" => "decrypt",
"message" => $text);
$package_json = json_encode($package);
$serverSays = transmit($package_json);
$serverSaysArray = json_decode($serverSays);
return base64_decode($serverSaysArray->decrypted_message);
}
function transmit($package) {
global $CRYPTO_PORT;
global $CRYPTO_HOST;
$serverLink = fsockopen($CRYPTO_HOST, $CRYPTO_PORT);
if ($serverLink === FALSE) {
error_log("Could not connect to encryption server");
return FALSE;
}
fwrite($serverLink, $package . "\n");
$response = '';
while (!feof($serverLink)) {
$response .= fgets($serverLink, 128);
}
fclose($serverLink);
return $response;
}
while (TRUE) {
$enc = encrypt('totsadaddywoopxxx');
print "$enc\n";
$dec = decrypt($enc);
print "$dec\n";
$enc = encrypt('totsadaddywoopxxx');
print "$enc\n";
$dec = decrypt($enc);
print "$dec\n";
$enc = encrypt('totsadaddywoopxxx');
print "$enc\n";
$dec = decrypt($enc);
print "$dec\n";
#print decrypt('1c6dee677126551fa4b3f0732986dc3b7c985c64c07075e3651213d7a69435bcd87083e729e8de860c');
#print "\n";
#print decrypt('550cbec7498371dc01bcd6b88fc623b47cb2efd1881da6e07ee992229308305992bbc7ccc374f00c91d56d10a68d6110e2');
print "===========================\n";
sleep(1);
}

In your decryption routine you use:
unhexed.split('|')
to find the boundary between IV and the ciphertext. However, the IV is generated randomly by the sender. Sometimes, one of its 16 bytes will be 124, that is the boundary character '|'.
When that happens (in roughly 6% of the cases), the decryption routine will initialize the cipher either with
an IV with length between 1 and 15 bytes, which leads to the exception, or
an IV with length 0, which leads to the incorrect decryption, because in PyCrypto versions<2.6 an all-zero 16 byte IV will be used by default
In the decryption routine you should have instead:
components = [ unhexed[:AES.block_size], unhexed[AES.block_size+1:] ]
Or you could get rid of the '|' separator altogether.

Related

how to convert AsymmetricKeyPair to base64 encoding string in dart

I‘m a rookie in PointyCastle,how can I generate ecc base64 key pairs with PointyCastle in dart
AsymmetricKeyPair<PublicKey, PrivateKey> generateKeyPair({curve = 'secp256r1'}) {
var param = curve == 'secp256r1' ? ECCurve_secp256r1() : ECCurve_secp256k1();
var keyParams = ECKeyGeneratorParameters(param);
var random = FortunaRandom();
random.seed(KeyParameter(_seed()));
var generator = ECKeyGenerator();
generator.init(ParametersWithRandom(keyParams, random));
return generator.generateKeyPair();
}
Uint8List _seed() {
var random = Random.secure();
var seed = List<int>.generate(32, (_) => random.nextInt(256));
return Uint8List.fromList(seed);
}
above I can generate a AsymmetricKeyPair object,how to get public base64 and private base64 like my already done js bundle
{"priKey":"CVK3r/UxdGCwQBjtn5vN/orUMxKf9E/1TlJzLkMz9t4=","pubKey":"BJH/mWJqgchGxXGA5/E79SsWRwVo3rpduBmD8FOs7UlKiK8PIvwrkCDvUcwhKdysW35OByPjoVcwFqg1NyumLKM="}
besides, at first I want to use my js bundle file in flutter android, but that would be very tough as I know
how to use js bundle in flutter android
I also need to make sign and verify function with ecdsa in sha256
Signer signer = new Signer('SHA-256/ECDSA');
// 签名,参数:私钥,明文
ECSignature sign(String privateKeyStr, String txt) {
SecureRandom random = _setRadom();
// how to convert a plain txt to kTestBytes
final Uint8List kTestBytes = new Uint8List.fromList([1, 2, 3]);
// if I pass in base64 privateKey str, is the radix 64?
ECPrivateKey privateKey = new ECPrivateKey(BigInt.parse(privateKeyStr, radix: 10), ecDomain);
PrivateKeyParameter signParams = new PrivateKeyParameter(privateKey);
signer.init(true, new ParametersWithRandom(signParams, random));
ECSignature signature = signer.generateSignature(kTestBytes) as ECSignature;
return signature;
}
// 验签
// 参数: 明文,签名,公钥
bool verify(txt, signature, publicKey) {
// how to make the txt to kTestBytes
var kTestBytes = txt
signer.init(false, new PublicKeyParameter(publicKey));
bool verify = signer.verifySignature(kTestBytes, signature);
// print(verify);
return verify;
}
// I don't know what this function actually do,as I know in other language ecdsa don;t need a random number.
SecureRandom _setRadom() {
List<int> key = [67, 3, 241, 75, 143, 78, 115, 99, 21, 242, 180, 43, 26, 7, 194, 20];
List<int> iv = [87, 117, 137, 182, 2, 199, 132, 230, 120, 12, 109, 177, 34, 197, 186, 206];
KeyParameter keyParam = new KeyParameter(new Uint8List.fromList(key));
ParametersWithIV<KeyParameter> paramsIV = new ParametersWithIV(keyParam, new Uint8List.fromList(iv));
SecureRandom random = new SecureRandom('AES/CTR/AUTO-SEED-PRNG')..seed(paramsIV);
return random;
}
Any place can I find sample codes, these functions were changed by some copies on the internet。and I got many confuses list in the comment
The EC keys are to be exported/imported as a Base64 encoded raw private key and as a Base64 encoded raw uncompressed public key.
First of all it can be stated that signing and verifying with the posted code works when a fresh key pair is generated:
AsymmetricKeyPair<PublicKey, PrivateKey> kp = generateKeyPair();
ECSignature signature = sign(kp.privateKey as ECPrivateKey, "The quick brown fox jumps over the lazy dog");
bool verified = verify("The quick brown fox jumps over the lazy dog", signature, kp.publicKey as ECPublicKey);
print(verified); // true
I.e. in the following I will focus on the import/export of the keys. The raw keys are encapsulated in ECPrivateKey and ECPublicKey and can be exported and imported as follows:
String p256 = 'secp256r1';
AsymmetricKeyPair<PublicKey, PrivateKey> kp = generateKeyPair(curve: p256);
// Export
BigInt d = (kp.privateKey as ECPrivateKey).d!;
BigInt x = (kp.publicKey as ECPublicKey).Q!.x!.toBigInteger()!;
BigInt y = (kp.publicKey as ECPublicKey).Q!.y!.toBigInteger()!;
// Import
ECDomainParameters domain = ECDomainParameters(p256);
ECPrivateKey ecPrivateKey = ECPrivateKey(d, domain);
ECPublicKey ecPublicKey = ECPublicKey(domain.curve.createPoint(x, y), domain);
Here d is the raw private key and x and y are the x and y coordinates of the raw public key.
On the JavaScript side, the Base64 encoded raw private key is used and the Base64 encoded raw uncompressed key, where the uncompressed key is the concatenation 0x04 + <x> + <y>. So, for export and import, the following applies:
Regarding the export, the Base64 encoded raw private and public key can be derived from d, x, and y as follows:
String rawPrivateB64 = exportPrivate(d);
String rawUncompressedPublicB64 = exportPublic(x, y, 32);
with
import 'dart:convert';
import 'package:nanodart/nanodart.dart';
String exportPrivate(BigInt d){
return base64Encode(NanoHelpers.bigIntToBytes(d));
}
String exportPublic(BigInt x, BigInt y, int size){
return base64Encode(NanoHelpers.concat([Uint8List.fromList([4]), pad(NanoHelpers.bigIntToBytes(x), size), pad(NanoHelpers.bigIntToBytes(y), size)]));
}
Uint8List pad(Uint8List list, int size){
Uint8List padded = list;
int currSize = list.length;
if (currSize < size){
Uint8List pad = Uint8List(size - currSize);
padded = NanoHelpers.concat([pad, list]);
}
return padded;
}
The size passed in exportPublic() as 3rd parameter is the size of the order of the generator point (32 bytes for secp256r1). If x or y are smaller, they are each padded from the front with 0x00 values until the required length is reached.
For the conversion from BigInt to Uint8List and the concatenation I used the NanoHelpers class from the NanoDart package for simplicity. Of course, other implementations can be used here as well.
Regarding the import, d, x, and y can be derived from the Base64 encoded raw private and public key as follows:
BigInt d = importPrivate(rawPrivateB64);
List xy = importPublic(rawUncompressedPublicB64);
BigInt x = xy.elementAt(0);
BigInt y = xy.elementAt(1);
with
BigInt importPrivate(String d){
return NanoHelpers.byteToBigInt(base64Decode(d));
}
List importPublic(String xy){
Uint8List xyBytes = base64Decode(xy);
xyBytes = xyBytes.sublist(1, xyBytes.length);
int size = xyBytes.length ~/ 2;
Uint8List x = xyBytes.sublist(0, size);
Uint8List y = xyBytes.sublist(size);
return [NanoHelpers.byteToBigInt(x), NanoHelpers.byteToBigInt(y)];
}
Test
The posted key pair
{"priKey":"CVK3r/UxdGCwQBjtn5vN/orUMxKf9E/1TlJzLkMz9t4=","pubKey":"BJH/mWJqgchGxXGA5/E79SsWRwVo3rpduBmD8FOs7UlKiK8PIvwrkCDvUcwhKdysW35OByPjoVcwFqg1NyumLKM="}
can be imported and used for signing and verification as follows:
String rawPrivateB64 = "CVK3r/UxdGCwQBjtn5vN/orUMxKf9E/1TlJzLkMz9t4=";
String rawUncompressedPublicB64 = "BJH/mWJqgchGxXGA5/E79SsWRwVo3rpduBmD8FOs7UlKiK8PIvwrkCDvUcwhKdysW35OByPjoVcwFqg1NyumLKM=";
BigInt d = importPrivate(rawPrivateB64);
List xy = importPublic(rawUncompressedPublicB64);
BigInt x = xy.elementAt(0);
BigInt y = xy.elementAt(1);
ECDomainParameters domain = ECDomainParameters('secp256r1');
ECPrivateKey private = ECPrivateKey(d, domain);
ECPublicKey public = ECPublicKey(domain.curve.createPoint(x, y), domain);
ECSignature signature = sign(private, "The quick brown fox jumps over the lazy dog");
bool verified = verify("The quick brown fox jumps over the lazy dog", signature, public);
print(verified); // true
Edit:
Regarding your comment: sign() and verify() are the methods you posted, but according to the changes, the keys are now passed directly (instead of strings) and the actual message is applied (instead of [1,2,3]) using utf8.encode() for UTF-8 encoding:
import 'dart:convert';
ECSignature sign(ECPrivateKey privateKey, String txt) {
SecureRandom random = _setRandom();
Uint8List txtBytes = Uint8List.fromList(utf8.encode(txt));
PrivateKeyParameter signParams = PrivateKeyParameter(privateKey);
signer.init(true, ParametersWithRandom(signParams, random));
ECSignature signature = signer.generateSignature(txtBytes) as ECSignature;
return signature;
}
bool verify(String txt, ECSignature signature, ECPublicKey publicKey) {
signer.init(false, PublicKeyParameter(publicKey));
bool verify = signer.verifySignature(Uint8List.fromList(utf8.encode(txt)), signature);
return verify;
}
As _setRandom() I applied the implementation from generateKeyPair() instead of your implementation (i.e. a FortunaRandom() based CSPRNG).
my whole code based on accepted answer
crypto.dart
import "dart:typed_data";
import "dart:math";
import 'dart:convert';
import "package:pointycastle/export.dart";
import './utils.dart';
Signer signer = new Signer('SHA-256/ECDSA');
class Crypto {
final ECDomainParameters ecDomain = new ECDomainParameters('secp256r1');
/// 公私钥对生成
/// 关于公私钥encoding:https://stackoverflow.com/questions/72641616/how-to-convert-asymmetrickeypair-to-base64-encoding-string-in-dart
///
Map generateKeyPair({curve = 'secp256r1'}) {
var param = curve == 'secp256r1' ? ECCurve_secp256r1() : ECCurve_secp256k1();
var keyParams = ECKeyGeneratorParameters(param);
var random = FortunaRandom();
random.seed(KeyParameter(this._seed(32)));
var generator = ECKeyGenerator();
generator.init(ParametersWithRandom(keyParams, random));
AsymmetricKeyPair<PublicKey, PrivateKey> kp = generator.generateKeyPair();
BigInt d = (kp.privateKey as ECPrivateKey).d!;
BigInt x = (kp.publicKey as ECPublicKey).Q!.x!.toBigInteger()!;
BigInt y = (kp.publicKey as ECPublicKey).Q!.y!.toBigInteger()!;
String rawPrivateB64 = exportPrivate(d);
String rawUncompressedPublicB64 = exportPublic(x, y, 32);
return {'base64Pub': rawUncompressedPublicB64, 'base64Priv': rawPrivateB64};
}
Uint8List _seed(size) {
var random = Random.secure();
var seed = List<int>.generate(size, (_) => random.nextInt(256));
return Uint8List.fromList(seed);
}
// TODO
// Restore the ECPrivateKey from 'd'.
restoreKeyFromPrivate(privateKeyStr) {
ECPrivateKey privateKey = new ECPrivateKey(BigInt.parse(privateKeyStr, radix: 10), ecDomain);
ECPoint Q = privateKey.parameters!.G * privateKey.d as ECPoint;
ECPublicKey publicKey = new ECPublicKey(Q, privateKey.parameters);
return publicKey;
}
ECSignature sign(ECPrivateKey privateKey, String txt) {
SecureRandom random = _setRandom();
Uint8List txtBytes = Uint8List.fromList(utf8.encode(txt));
PrivateKeyParameter signParams = PrivateKeyParameter(privateKey);
signer.init(true, ParametersWithRandom(signParams, random));
ECSignature signature = signer.generateSignature(txtBytes) as ECSignature;
return signature;
}
bool verify(String txt, ECSignature signature, ECPublicKey publicKey) {
signer.init(false, PublicKeyParameter(publicKey));
bool verify = signer.verifySignature(Uint8List.fromList(utf8.encode(txt)), signature);
return verify;
}
SecureRandom _setRandom() {
// List<int> key = [67, 3, 241, 75, 143, 78, 115, 99, 21, 242, 180, 43, 26, 7, 194, 20];
// List<int> iv = [87, 117, 137, 182, 2, 199, 132, 230, 120, 12, 109, 177, 34, 197, 186, 206];
List<int> key = _seed(16);
List<int> iv = _seed(16);
KeyParameter keyParam = new KeyParameter(new Uint8List.fromList(key));
ParametersWithIV<KeyParameter> paramsIV = new ParametersWithIV(keyParam, new Uint8List.fromList(iv));
SecureRandom random = new SecureRandom('AES/CTR/AUTO-SEED-PRNG')..seed(paramsIV);
return random;
}
}
crypto_test.dart
import '../lib/crypto/crypto.dart';
import "dart:typed_data";
import "dart:math";
import '../lib/crypto/utils.dart';
import "package:pointycastle/export.dart";
var crypto = new Crypto();
void main() {
// _generateTest();
// _signTest();
Map keyPair = crypto.generateKeyPair();
String rawPrivateB64 = keyPair['base64Priv'];
String rawUncompressedPublicB64 = keyPair['base64Pub'];
BigInt d = importPrivate(rawPrivateB64);
List xy = importPublic(rawUncompressedPublicB64);
BigInt x = xy.elementAt(0);
BigInt y = xy.elementAt(1);
ECDomainParameters domain = ECDomainParameters('secp256r1');
ECPrivateKey private = ECPrivateKey(d, domain);
ECPublicKey public = ECPublicKey(domain.curve.createPoint(x, y), domain);
ECSignature signature = crypto.sign(private, "The quick brown fox jumps over the lazy dog");
bool verified = crypto.verify("The quick brown fox jumps over the lazy dog", signature, public);
print('验签结果');
print(verified); // true
}
_generateTest() {
Map keyPair = crypto.generateKeyPair();
print("结果");
print(keyPair['base64Pub']);
print(keyPair['base64Priv']);
}
_signTest() {
// AsymmetricKeyPair<PublicKey, PrivateKey> kp = crypto.generateKeyPair();
// ECSignature signature = sign(kp.privateKey as ECPrivateKey, "The quick brown fox jumps over the lazy dog");
// bool verified = crypto.verify("The quick brown fox jumps over the lazy dog", signature, kp.publicKey as ECPublicKey);
// print(verified); // true
}

Cannot create hdf5 file with h5py.file()

I'm trying to write my results in two separate hdf5 files. My code works for one of them but produces an error for the other one, even though the code for both is identical. Here is my code:
def test(model, outfile_f3,outfile_f5):
dtype = torch.FloatTensor
path = args.imagefolder
filenames = sorted(glob.glob(os.path.join(path, '*.png')))
N = len(filenames)
f3_activations = []
f5_activations = []
for i in range(N):
fname = filenames[i]
print(fname)
v = Image.open(fname)
image = load_image(v, image_sizes['eig'])
image = torch.from_numpy(image).type(dtype).cuda()
image = image.unsqueeze(0)
att, f3, f5, latent = model(image, segment=args.segment, add_offset=args.addoffset and args.segment, test=True)
#latents.append(latent.detach()[0].cpu().numpy().flatten())
#attended.append(att.detach()[0].cpu().numpy().flatten())
f3_activations.append(f3.detach()[0].cpu().numpy().flatten())
f5_activations.append(f5.detach()[0].cpu().numpy().flatten())
file_f3 = h5py.File(outfile_f3, 'w')
file_f3.create_dataset('f3_activations', data=np.array(f3_activations))
file_f3.close()
file_f5 = h5py.File(outfile_f5, 'w')
file_f5.create_dataset('f5_activations', data=np.array(f5_activations))
file_f5.close()
#asciiList = [n.split('/')[-1][:-4].encode("ascii", "ignore") for n in filenames]
#f.create_dataset('filenames', (len(asciiList), 1), 'S10', data=asciiList)
outfile_f3 and outfile_f5 have previously been defined as:
outfile_f3 = os.path.join('./output', 'f3_activations.hdf5')
outfile_f5 = os.path.join('.output','f5_activations.hdf5')
When I try to run the script that uses test function I get the following error:
File "record_f3_and_f5_actvations.py", line 109, in test
file_f5 = h5py.File(outfile_f5, 'w')
File "/gpfs/milgram/project/yildirim/ms3542/conda_envs/env/lib/python3.6/site-packages/h5py/_hl/files.py", line 408, in init
swmr=swmr)
File "/gpfs/milgram/project/yildirim/ms3542/conda_envs/env/lib/python3.6/site-packages/h5py/_hl/files.py", line 179, in make_fid
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "h5py/h5f.pyx", line 108, in h5py.h5f.create
OSError: Unable to create file (unable to open file: name = '.output/f5_activations.hdf5', errno = 2, error message = 'No such file or directory', flags = 13, o_flags = 242)
It has no problem with creating, opening and writing in the file f3_activations.hdf5, but for some reason, it can't do the same with f5_activations.hdf5

Is there any way to save mulaw audio stream from twilio in a file

I am using Twilio voice stream feature and i don't want to use Twilio record functionality. When Twilio starts sending voice stream to my server i want to store it into disk as an audio file in realtime.
I was running into the same issue today and figured a way to generate a WAVE Header for the mu-law header:
If you're following Twilio's blog post, that's the code I ended implementing:
wss.on('connection', (socket) => {
socket.on('message', (msg) => {
const { event, ...message } = JSON.parse(msg);
switch (event) {
case 'start':
let streamSid = message.start.streamSid;
socket.wstream = fs.createWriteStream(__dirname + `/${Date.now()}.wav`, { encoding: 'binary' });
// This is a mu-law header for a WAV-file compatible with twilio format
socket.wstream.write(Buffer.from([
0x52,0x49,0x46,0x46,0x62,0xb8,0x00,0x00,0x57,0x41,0x56,0x45,0x66,0x6d,0x74,0x20,
0x12,0x00,0x00,0x00,0x07,0x00,0x01,0x00,0x40,0x1f,0x00,0x00,0x80,0x3e,0x00,0x00,
0x02,0x00,0x04,0x00,0x00,0x00,0x66,0x61,0x63,0x74,0x04,0x00,0x00,0x00,0xc5,0x5b,
0x00,0x00,0x64,0x61,0x74,0x61,0x00,0x00,0x00,0x00, // Those last 4 bytes are the data length
]));
break;
case 'media':
// decode the base64-encoded data and write to stream
socket.wstream.write(Buffer.from(message.media.payload, 'base64'));
break;
case 'stop':
// Now the only thing missing is to write the number of data bytes in the header
socket.wstream.write("", () => {
let fd = fs.openSync(socket.wstream.path, 'r+'); // `r+` mode is needed in order to write to arbitrary position
let count = socket.wstream.bytesWritten;
count -= 58; // The header itself is 58 bytes long and we only want the data byte length
console.log(count)
fs.writeSync(
fd,
Buffer.from([
count % 256,
(count >> 8) % 256,
(count >> 16) % 256,
(count >> 24) % 256,
]),
0,
4, // Write 4 bytes
54, // starts writing at byte 54 in the file
);
});
break;
}
});
});
You can use FFmpeg to convert the Twilio mulaw to a regular WAV file.
If you use the class below, then you will just need to send the Twilio stream buffers when they arrive.
Something like:
recording = StreamAudioRecording(tempDirectory)
recording.start_recording(call_id)
<loop over buffer packets arriving>
recording.write_buffer(buffer)
recording_audio_path = recording.stop_recording()
This is the class implementation
import os
RAW_AUDIO_FILE_EXTENSION = "ulaw"
CONVERTED_AUDIO_FILE_EXTENSION = "wav"
class StreamAudioRecording:
def __init__(self, audio_recording_path):
self.audio_recording_path = audio_recording_path
self.f = None
self.audio_file_path = None
def start_recording(self, call_id):
self.audio_file_path = os.path.join(self.audio_recording_path, f" .
{call_id}.{RAW_AUDIO_FILE_EXTENSION}")
self.f = open(self.audio_file_path, 'wb')
def write_buffer(self, buffer):
self.f.write(buffer)
def stop_recording(self):
self.f.close()
converted_audio_path =
self.audio_file_path.replace(RAW_AUDIO_FILE_EXTENSION,
CONVERTED_AUDIO_FILE_EXTENSION)
self.convert_call_recording(self.audio_file_path, converted_audio_path)
return converted_audio_path
#staticmethod
def convert_call_recording(mulaw_path, wav_path):
command = f"ffmpeg -y -loglevel panic -f mulaw -ar 8k -ac 1 -i {mulaw_path} {wav_path}"
os.system(command)
If your using Node.js, the solution that #tdeo provided works great. I was inspired and I made a simple library using this solution. It's available now on npm.
https://www.npmjs.com/package/twilio-media-stream-save-audio-file

model = CSRNet() | TypeError: super(type, obj): obj must be an instance or subtype of type

transform=transforms.Compose([
transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
root = '/home/meet/CSRNet-pytorch/ShanghaiTech/'
part_A_train = os.path.join(root,'part_A/train_data','images')
part_A_test = os.path.join(root,'part_A/test_data','images')
part_B_train = os.path.join(root,'part_B/train_data','images')
part_B_test = os.path.join(root,'part_B/test_data','images')
path_sets = [part_A_test]
#defining the image path
img_paths = []
for path in path_sets:
for img_path in glob.glob(os.path.join(path, '*.jpg')):
img_paths.append(img_path)
model = CSRNet()
model = model.cuda()
On running model = CSRNet() I get an "Typeerror: super(type, obj): obj must be an instance or subtype of type"
I am referring to a tutorial to get the number of people count from crowd images.
This is the tutorial I am referring to:
https://www.analyticsvidhya.com/blog/2019/02/building-crowd-counting-model-python/
This is the kernal output window : -
model = CSRNet()
Downloading: "https://download.pytorch.org/models/vgg16-397923af.pth" to /home/meet/.torch/models/vgg16-397923af.pth
Traceback (most recent call last):
File "", line 1, in
model = CSRNet()
File "/home/meet/CSRNet-pytorch/model.py", line 16, in init
mod = models.vgg16(pretrained = True)
File "/home/meet/anaconda3/lib/python3.7/site-packages/torchvision/models/vgg.py", line 152, in vgg16
model.load_state_dict(model_zoo.load_url(model_urls['vgg16']))
File "/home/meet/anaconda3/lib/python3.7/site-packages/torch/utils/model_zoo.py", line 66, in load_url
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
File "/home/meet/anaconda3/lib/python3.7/site-packages/torch/utils/model_zoo.py", line 73, in _download_url_to_file
u = urlopen(url, stream=True)
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/api.py", line 70, in get
return request('get', url, params=params, **kwargs)
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/api.py", line 55, in request
with sessions.Session() as session:
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 346, in init
self.mount('https://', HTTPAdapter())
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 117, in init
super(HTTPAdapter, self).init()

Reading the fileset from a torrent

I want to (quickly) put a program/script together to read the fileset from a .torrent file. I want to then use that set to delete any files from a specific directory that do not belong to the torrent.
Any recommendations on a handy library for reading this index from the .torrent file? Whilst I don't object to it, I don't want to be digging deep into the bittorrent spec and rolling a load of code from scratch for this simple purpose.
I have no preference on language.
I would use rasterbar's libtorrent which is a small and fast C++ library.
To iterate over the files you could use the torrent_info class (begin_files(), end_files()).
There's also a python interface for libtorrent:
import libtorrent
info = libtorrent.torrent_info('test.torrent')
for f in info.files():
print "%s - %s" % (f.path, f.size)
Effbot has your question answered. Here is the complete code to read the list of files from .torrent file (Python 2.4+):
import re
def tokenize(text, match=re.compile("([idel])|(\d+):|(-?\d+)").match):
i = 0
while i < len(text):
m = match(text, i)
s = m.group(m.lastindex)
i = m.end()
if m.lastindex == 2:
yield "s"
yield text[i:i+int(s)]
i = i + int(s)
else:
yield s
def decode_item(next, token):
if token == "i":
# integer: "i" value "e"
data = int(next())
if next() != "e":
raise ValueError
elif token == "s":
# string: "s" value (virtual tokens)
data = next()
elif token == "l" or token == "d":
# container: "l" (or "d") values "e"
data = []
tok = next()
while tok != "e":
data.append(decode_item(next, tok))
tok = next()
if token == "d":
data = dict(zip(data[0::2], data[1::2]))
else:
raise ValueError
return data
def decode(text):
try:
src = tokenize(text)
data = decode_item(src.next, src.next())
for token in src: # look for more tokens
raise SyntaxError("trailing junk")
except (AttributeError, ValueError, StopIteration):
raise SyntaxError("syntax error")
return data
if __name__ == "__main__":
data = open("test.torrent", "rb").read()
torrent = decode(data)
for file in torrent["info"]["files"]:
print "%r - %d bytes" % ("/".join(file["path"]), file["length"])
Here's the code from Constantine's answer above, slightly modified to handle Unicode characters in torrent filenames and fileset filenames in torrent info:
import re
def tokenize(text, match=re.compile("([idel])|(\d+):|(-?\d+)").match):
i = 0
while i < len(text):
m = match(text, i)
s = m.group(m.lastindex)
i = m.end()
if m.lastindex == 2:
yield "s"
yield text[i:i+int(s)]
i = i + int(s)
else:
yield s
def decode_item(next, token):
if token == "i":
# integer: "i" value "e"
data = int(next())
if next() != "e":
raise ValueError
elif token == "s":
# string: "s" value (virtual tokens)
data = next()
elif token == "l" or token == "d":
# container: "l" (or "d") values "e"
data = []
tok = next()
while tok != "e":
data.append(decode_item(next, tok))
tok = next()
if token == "d":
data = dict(zip(data[0::2], data[1::2]))
else:
raise ValueError
return data
def decode(text):
try:
src = tokenize(text)
data = decode_item(src.next, src.next())
for token in src: # look for more tokens
raise SyntaxError("trailing junk")
except (AttributeError, ValueError, StopIteration):
raise SyntaxError("syntax error")
return data
n = 0
if __name__ == "__main__":
data = open("C:\\Torrents\\test.torrent", "rb").read()
torrent = decode(data)
for file in torrent["info"]["files"]:
n = n + 1
filenamepath = file["path"]
print str(n) + " -- " + ', '.join(map(str, filenamepath))
fname = ', '.join(map(str, filenamepath))
print fname + " -- " + str(file["length"])
bencode.py from the original Mainline BitTorrent 5.x client (http://download.bittorrent.com/dl/BitTorrent-5.2.2.tar.gz) would give you pretty much the reference implementation in Python.
It has an import dependency on the BTL package but that's trivially easy to remove. You'd then look at bencode.bdecode(filecontent)['info']['files'].
Expanding on the ideas above, I did the following:
~> cd ~/bin
~/bin> ls torrent*
torrent-parse.py torrent-parse.sh
~/bin> cat torrent-parse.py
# torrent-parse.py
import sys
import libtorrent
# get the input torrent file
if (len(sys.argv) > 1):
torrent = sys.argv[1]
else:
print "Missing param: torrent filename"
sys.exit()
# get names of files in the torrent file
info = libtorrent.torrent_info(torrent);
for f in info.files():
print "%s - %s" % (f.path, f.size)
~/bin> cat torrent-parse.sh
#!/bin/bash
if [ $# -lt 1 ]; then
echo "Missing param: torrent filename"
exit 0
fi
python torrent-parse.py "$*"
You'll want to set permissions appropriately to make the shell script executable:
~/bin> chmod a+x torrent-parse.sh
Hope this helps someone :)

Resources