On Software Reverse Engineering

#include

"l_strkey.h"

... ...

int main(int argc, char

**argv)

{

... ...

strcpy(outfile,

"lm_new.c");

... ...

/* initialize job, could report

error and exit here */

if (lc_init(0, vendor_name, &vendorkeys[0], &job))

{

fprintf(stderr,

"lc_init failed: %s\n", lc_errstring(job));

exit(1);

}

/* random number generation via

sb_rngFIPS186Session() */

l_genrand(job, lmseed1,

lmseed2, lmseed3, NEWSEEDSIZ, newseeds);

... ... /* convert newseeds into

seed1 – seed4, and so on */

/* write the four ENCRYPTION_SEED

to lmseeds.h */

if (!(ofp = fopen("lmseeds.h", "w")))

{

perror("Can't

write lmseeds.h file, exiting");

exit(1);

}

fprintf(ofp, "...

...", seed1, seed2, seed3, seed4);

fclose(ofp);

/* now it’s lm_new.c */

if (!(ofp = fopen(outfile, "w")))

{

perror("Can't

open output file, exiting");

exit(1);

}

... ...

/* make sure default and weak seeds

are excluded */

if (!l_reasonable_seed(seed3) || !l_reasonable_seed(seed4) ||

!l_reasonable_seed(lmseed1)

|| !l_reasonable_seed(lmseed2) ||

!l_reasonable_seed(lmseed3))

{

... ...

fprintf(stderr,

"Existing.\n");

exit(1);

}

... ...

fputs("#include

<time.h>\n", ofp);

do_real(); /* write the main content of lm_new.c */

fclose(ofp);

return 0;

}

static void do_real()

{

... ... /* generate random variable

and function names */

while (!key5_uniqx)

{

key5_uniqx =

our_rand(256) + (our_rand(256) << 8) +

(our_rand(256) << 16) + (our_rand(256)

<< 24);

}

... ... /* generate random

key5_order[] */

for (i = 0; i < 200; i += 2)

random_garbage();

... ... /* more random garbage */

for (counter=0; counter<keysize; counter++)/* generally keysize=1, only one vendor */

{

key5(&vendorkeys[counter]); /* obfuscate vendorkeys->data[] */

l_xorname(vname,

&vendorkeys[counter]);/* obfuscate vendorkeys->keys[]

/* real VENDORCODE initialization */

do_ulong(vendorkeys[counter].data[0],

"data[0]", counter);

do_ulong(vendorkeys[counter].data[1],

"data[1]", counter);

do_ulong(vendorkeys[counter].keys[0],

"keys[0]", counter);

do_ulong(vendorkeys[counter].keys[1],

"keys[1]", counter);

do_ulong(vendorkeys[counter].keys[2],

"keys[2]", counter);

do_ulong(vendorkeys[counter].keys[3],

"keys[3]", counter);

... ...

}

l_puts_rand(ofp, fpVar,

numvars); /* output lines in random order */

fflush(ofp);

uniqcode(); /* output the source of lm_new.c!l_n36_buff() */

fflush(ofp);

... ...

}

static void

do_ulong(unsigned long ul, char *varname, int counter)

{

... ...

randvarname(b1,

"b1"); bnames[0] = b1; /* generate junk

variable names */

randvarname(b2,

"b2"); bnames[1] = b2;

randvarname(b3,

"b3"); bnames[2] = b3;

randvarname(b4,

"b4"); bnames[3] = b4;

for (i = 0; i < 4; i++)

{

shift = i * 8;

CLEARV; /* break up ul into 4 bytes and assign them to 4 junk-named

variables */

sprintf(vBuf,

"static unsigned int %s = %d;\n",

bnames[i], (ul & (0xff << shift))

>> shift );

fwrite(vBuf,

sizeof(char), sizeof(vBuf), fpVar);

CLEARF; /* reassemble the 4 variables back to let v->varname =

ul */

sprintf(fBuf,

"\tif (%s == %d) v->%s += (%s << %d);\n",

countervar, counter, varname, bnames[i],

shift);

fwrite(fBuf,

sizeof(char), sizeof(fBuf), fpFunc);

}

static void uniqcode()

{

... ...

int idx = *vendor_name % XOR_SEEDS_ARRAY_SIZ; /* idx = V % 20 = 86 % 20 = 6 */

XOR_SEEDS_INIT_ARRAY(xor_arr)

/* substitution table defined in l_strkey.h */

fprintf(ofp,

"static void %s(job, vendor_id, key) \n

... ...

unsigned long x

= 0x%x; \n

struct s_tmp

{int i; char *cp; unsigned char a[12];} *t, t2; \n

if (job) t =

(struct s_tmp *)job; \n

else t =

&t2; \n

... ...",

key5_fname, key5_uniqx); /* key5_fname and key5_uniqx

are all random */

for(i = 0; i < SEEDS_XOR_NUM; i++) /* decode t->a[], i.e. job->mem_ptr2_bytes[] */

{

unsigned char num;

cpp[i] = cp;

sprintf(cp,

"t->cp=(char *)(((long)t->cp) ^ (time(0) ^ ((0x%x << 16) +

0x%x)));\n

t->a[%d] = (time(0) & 0xff) ^

0x%x;\n", /* runtime randomness */

our_rand(0xff), our_rand(0xff), i,

our_rand(0xff));

cp += strlen(cp) +

}

l_puts_rand1(ofp,

SEEDS_XOR_NUM, cpp); /* output lines in random order

fprintf(ofp, "for

(i = 0; i < %d; i++) \n

{ \n

(sig[i%%SIGSIZE] != vendor_id[i%%len]) \n

sig[i%%SIGSIZE] ^= vendor_id[i%%len]; \n

} \n

unsigned long y =

((((long)sig[0] << %d) \n

| ((long)sig[1] << %d) \n

| ((long)sig[2] << %d) \n

| ((long)sig[3] << %d)) \n

^ ((long)(t->a[%d]) << 0) \n

^ ((long)(t->a[%d]) << 8) \n

^ ((long)(t->a[%d]) << 16) \n

^ ((long)(t->a[%d]) << 24) \n

^ x \n

^ key->keys[0] \n

^ key->keys[1]) & 0xffffffff; \n

key->data[0] ^= y;

key->data[1] ^= y;

... ...",

MAX_DAEMON_NAME, /* MAX_DAEMON_NAME = 10 defined in

lmclient.h */

key5_order[0],

key5_order[1], key5_order[2], key5_order[3],

xor_arr[idx][0],

xor_arr[idx][1], xor_arr[idx][2], xor_arr[idx][3]));

}

static void

key5(VENDORCODE *k) /* obfuscate encryption seeds,

i.e. k->data[] */

{

... ...

/* same key5_uniqx, key5_order[],

sig[] and MAX_DAEMON_NAME as in uniqcode() */

for (i = 0; i < MAX_DAEMON_NAME; i++)

{

if (sig[i%SIGSIZE]

!= vname[i % len])

sig[i%SIGSIZE]

^= vname[i % len];

}

y = ((((long)sig[0] << key5_order[0])

| ((long)sig[1] << key5_order[1])

| ((long)sig[2] << key5_order[2])

| ((long)sig[3] << key5_order[3]))

^ key5_uniqx

^ k->keys[0]

^ k->keys[1]) & 0xffffffff;

k->data[0] ^= y; /* k->data[0] = ENCRYPTION_SEED1 ^ y */

k->data[1] ^= y; /* k->data[1] = ENCRYPTION_SEED2 ^ y */

}

It’s a little long, but not incomprehensible once we

unravel its structure. The main procedure spends much of its time working on lmseeds.h and lm_new.c is

largely procreated in do_real(). With the help of several

subroutines do_real() per se focused on making

l_n36_buf() while

left l_n36_buff() to uniqcode(). And what about l_xorname()?

Although not appearing in l_n36_buf() as is, it’s right there

in do_real(). Therefore it is true that 4 vendor keys are

ciphered and deciphered by the same xor operations at different locations,

namely do_real() / l_n36_buf() and l_good_lic_key()

respectively.

Note lm_new.c itself is heavily obfuscated:

there are a lot of garbage code that are wholly useless, the identifiers are

random and meaningless, the real and trash code look alike and mix together,

the line order of real code are perturbed… Apparently it’s designed in this way

to fool the readers and it works, it’s almost impossible to understand the C

source, let alone the disassemblies. That’s why we need to concentrate on its

parent lmnewgen.c, which can gives us more hints.

For example, l_xorname() is called in do_real(), not l_n36_buf(). It

means key obfuscation is done in-house at vendor site and encoded keys are

linked into the target delivered to end users; then they are deciphered at user

runtime in l_good_lic_key(). Such scheme is intended to

minimize the exposure of real keys.

But we are more concerned about encryption seeds than

vendor keys. Recall it is job->mem_ptr2_bytes[] and code->data[] that are

different in the checkout and keygen processes. It is likely that FLEXlm also

uses xor for seed encoding, but it’s got to be more than that because of the

randomness we saw at user runtime. We’ll analyze it step by step.

Just one line above do_real()->l_xorname() there is

call to key5(), which obfuscates the raw

encryption seeds at vendor side (nothing is done to job structure). The vendor

key experience tells us that they must be de-obfuscated somewhere at user side.

This time it’s not l_good_lic_key(), but l_n36_buff(). Its

source in lm_new.c is readable, but still we

prefer to study its generator uniqcode(). Before long we find out that l_n36_buff() xor-es

the same variables as in key5(), plus the extra t->a[] stuff.

What’s that? Well t->a[] is just another name for job->mem_ptr2_bytes[]. How to

account for those additional xor? Let’s rewind several pages and… yes, at the L_MOVELONG() macros

in l_string_key(), right before the hashing starts.

Now we see the seed encryption/decryption is done in two

steps, l_n36_buf()/l_n36_buff() and l_n36_buff()/

l_string_key(). The xor code in the corresponding functions mirrors each

other, ensuring all noises are canceled out. The runtime randomness is

introduced by time(0) factor in l_n36_buff(), it

affects both code->data[] and job->mem_ptr2_bytes[] we saw

in l_string_key(). Indeed FLEXlm hides the encryption seeds in a

much more obscure way than vendor keys. And we now know the meaning of the two

weird-named functions (actually the only two) in lm_new.c: l_n36_buf() –

initialize VENDORCODE with ciphered seeds and keys; l_n36_buff() – undo l_n36_buf() and

impose the second step encryption.

We should emphasize that obfuscation is only for the

checkout process. In keygen process lmcrypt.exe does not

invoke l_n36_buf(), l_n36_buff(), or l_good_lic_key(), and L_MOVELONG has no

effects on raw seeds because job->mem_ptr2_bytes[] is

always 0. During checkout encryption/decryption there are a number of

constants, variables and arrays involved in the xor operations. They are often

placed at two locations, one for encoding and the other for decoding. A summary

about them would certainly be convenient.

Related objects

Encode

Decode

VENDOR_KEY

l_xorname(),

VENDORNAME,

VENDORMAGIC_V7

do_real(),

l_n36_buf()

l_good_lic_key()

ENCRYPTION_SEED,

step 1

x = key5_uniqx,

key5_order[],

sig[],

MAX_DAEMON_NAME

do_real(), key5(),

l_n36_buf()

uniqcode(),

l_n36_buff()

ENCRYPTION_SEED,

step 2

idx,

xor_arr

uniqcode(),

l_n36_buff()

L_MOVELONG(),

l_string_key()

It is time for us to retrieve the real encryption seeds

from cmath.exe. The true seeds are recovered as the first

argument for L_MOVELONG(). With idx=6 and the

prescribed substitution table xor_arr[], the xor operand consists of 4

bytes indexed at 7, 3, 5, 11 from job->mem_ptr2_bytes[]. After

assembling it becomes 00A0A000. Therefore

ENCRYPTION_SEED1

= code->data[0] ^ 00A0A000 =

52ED15B8 ^ 00A0A000 = 524DB5B8

ENCRYPTION_SEED2

= code->data[1] ^ 00A0A000 =

75CF780F ^ 00A0A000 = 756FD80F

Try them in lmseeds.h and…

bingo! lmcrypt.exe yields the lovely 6D5C01FD71C9.

Change the version to 5.0 and we get 3F23BE3056E4, correct signature again!

This assures us that doubtlessly we have acquired the authentic encryption

seeds and vendor keys of Visual Numerics. Finally we can generate any VNI

license keys as we wish, for other versions, other features, other products…

And hopefully it should work without a glitch provided FLEXlm is not materially

revised.

F:\flexlm>type

license.dat

FEATURE CMATH VNI 5.0