I have seen example of Cryptopp, there is GenerateRSAKey function which generate a public and private key to file, but I want to generate it to memory. Thus I have change the source code from
char* CRSAService::EncryptString(const char *pszPublicKey, char *pszSeeds, char *pszMessage)
{
char *pszResult;
RandomPool randPool;
randPool.IncorporateEntropy((byte *)pszSeeds, strlen(pszSeeds));
StringSource pubString(pszPublicKey, true, new HexDecoder);
RSAES_OAEP_SHA_Encryptor pub(pubString);
string szResult;
StringSource(pszMessage, true, new PK_EncryptorFilter(randPool, pub, new HexEncoder(new StringSink(szResult))));
pszResult = new char[szResult.size() + 1];
copy(szResult.begin(), szResult.end(), pszResult);
pszResult[szResult.size()] = '\0';
return pszResult;
}
char* CRSAService::DecryptString(const char *pszPrivateKey, char *pszCipherText)
{
char *pszResult;
StringSource privString(pszPrivateKey, true, new HexDecoder);
RSAES_OAEP_SHA_Decryptor priv(privString);
string szResult;
StringSource(pszCipherText, true, new HexDecoder(new PK_DecryptorFilter(GlobalRNG(), priv, new StringSink(szResult))));
pszResult = new char[szResult.size() + 1];
copy(szResult.begin(), szResult.end(), pszResult);
pszResult[szResult.size()] = '\0';
return pszResult;
}