On Software Reverse Engineering
FLEXlm Architecture
Equipped with VS + FLEXlm source, W32Dasm + cmath.exe, and IDA
+ cmath.exe (with signature), now we should be able to unveil
the FLEXlm kernel. The followings are some of our findings, where lm_ckout.c!lc_checkout() means
“function lc_checkout() in the module/file lm_ckout.c” and the
arrow symbol denotes function call. Notice that due to branching only portions
of the code are traced through and presented, but in general we are only
interested in those branches anyway.
0047D0C6: push 00000018 ;program entry point
0047D22D: call 00401000 ;call cmath.exe!main()
0047D240: call 0047F20D ;call chain to ntdll.dll!NtTerminateProcess()
sub_401000:
cmath.exe!main()
0040101D: call 004033B0 ;call vc++\flexlm.obj!imsl_f_lin_sol_gen()
00401039: call 00401050 ;call vc++\fwrimat.obj!imsl_f_write_matrix()
sub_004033B0:
vc++\flexlm.obj!imsl_f_lin_sol_gen()
004033C8: call 00408F24 ;call vc++\error.obj!imsl_e1psh()
0040342C: call 004034A0 ;call vc++\flinslg.obj!l_lin_sol_gen() to do the real work
sub_00408F24:
vc++\error.obj!imsl_e1psh()
00408F3A: call 0040A850 ;call chain vc++\single.obj!imsl_once() ® vc++\error.obj!l_error_init()
;® vc++\flexlm.obj!imsl_flexlm()
00408F76: call 00414AFD ;call
imsl_highland_check() ®
l_check.c!lc_timer() ®
l_timer_heart()
;® l_check() ® l_reconnect() ®
lm_ckout.c!l_checkout() as heartbeat
sub_00413290:
vc++\flexlm.obj!imsl_flexlm()
004132EF: call 004294A0 ;call lm_njob.c!lc_new_job()
004133A4: call 00426380 ;set LM_A_DISABLE_ENV to 1
004133DE: call 00426380 ;set LM_A_LICENSE_FILE_PTR to license file location
00413486: call 00426380 ;set LM_A_CHECK_INTERVAL to -1
004134C0: call 00426380 ;set
LM_A_RETRY_INTERVAL to -1
004134FB: call 00426380 ;set LM_A_RETRY_COUNT to -1
004135A7: call 00426380 ;set LM_A_LINGER to 0
004136A6: call 0042420C ;call l_check.c!lc_status(), returns LM_NEVERCHECKOUT
004138CD: call 0041A010 ;call lm_ckout.c!lc_checkout()
00414099: call 0047FA9B ;returns current date and time
004141C6: call 0042563D ;call lm_config.c!lc_get_config()
0041434E: call 0047F8F0 ;check if license is expired, returns 0 if not
sub_0041A010:
lm_ckout.c!lc_checkout()
0041A093: call 0041A14B ;call lm_ckout.c!l_checkout(), returns FFFFFFF8
sub_0041A14B:
lm_ckout.c!l_checkout()
0041A2E8: call [004AA01C] ;call lm_ckout.c!lm_start_real(), returns FFFFFFF8
sub_0041A875:
lm_ckout.c!lm_start_real()
0041AA47: call 0041B4A5 ;call lm_ckout.c!l_local_verify_conf(), returns 1=success
0041AC01: call 0041BD89 ;call lm_ckout.c!l_good_lic_key(), returns 0=failure
sub_0041BD89:
lm_ckout.c!l_good_lic_key()
0041BE30: call 00433D15 ;call l_getattr.c!l_xorname()
0041BE4D: call 0041DB9E ;call lm_ckout.c!l_sg()
0041C202: call 0041EBE3 ;call vc++\lm_ckout.obj!l_crypt_private(), returns 0
sub_0041DB9E:
lm_ckout.c!l_sg()
0041DBF7: call [004AD064] ;call lm_new.c!l_n36_buff()
0041DC16: call 00443283 ;call l_key.c!l_key()
sub_0041EBE3:
vc++\lm_ckout.obj!l_crypt_private()
0041EC07: call 0041EE44 ;call vc++\lm_ckout.obj!real_crypt(), returns 0
sub_0041EE44:
vc++\lm_ckout.obj!real_crypt()
0041F9B6: call 00420AF6 ;call vc++\lm_ckout.obj!l_string_key(), returns 0
sub_00420AF6:
vc++\lm_ckout.obj!l_string_key()
00420E94 - 00421156 ;invoke macro XOR_SEEDS_INIT_ARRAY(xor_arr)
00421247: call 0047F250 ;call strcpy(lkey, license_key)
0042145E: call 004803A0 ;call memcpy(newinput, input, inputlen)
0042191D: call 00421D66 ;call l_strkey.c!our_encrypt()
00421A13 - 00421B27 ;for{} loop license key matching
00421B34: call 00421C22 ;call l_strkey.c!atox() to convert binary string to ASCII
text
sub_00426380:
lm_set_attr.c!lc_set_attr()
004263E4: call 0042641E ;call lm_set_attr.c!l_set_attr() to set attributes in
config structure
sub_0042641E:
lm_set_attr.c!l_set_attr()
00427045: call 00427DBC ;if setting LM_A_LICENSE_FILE_PTR, call
lm_set_attr.c!l_set_license_path()
;® lm_config.c!l_flush_config() ® l_init_file.c!l_init_file()
;® l_allfeat.c!l_allfeat() ® l_allfeat.c!l_parse_feature_line()
;® l_allfeat.c!oldkey() ® vc++\l_allfeat.obj!l_crypt_private()
sub_004294A0:
lm_njob.c!lc_new_job()
004294C0: call [004A5A98] ;call lm_new.c!l_n36_buf(), returns 1
004294D2: call [004A5A98] ;call lm_new.c!l_n36_buf() with all 0 arguments, returns 0
004294E8: call 0044357F ;call chain lm_init.c!lc_init() ® lm_init.c!l_init()
004294FD – 00429516 ;turn on LM_OPTFLAG_CUSTOM_KEY5 flag
sub_0043873B:
vc++\l_allfeat.obj!l_crypt_private()
0043875F: call 00438771 ;call vc++\l_allfeat.obj!real_crypt(), returns 21D5B6E8572E
sub_00438771:
vc++\l_allfeat.obj!real_crypt()
004392DC: call 0043A41C ;call vc++\l_allfeat.obj!l_string_key(), returns
21D5B6E8572E
sub_0043A41C:
vc++\l_allfeat.obj!l_string_key()
sub_0044359E:
lm_init.c!l_init() ;take in
VENDORCODE and VENDORNAME to initialize job structure
00443E8F – 00444354 ;some validity test, could report error
004441F9: call 0041DB9E ;call lm_ckout.c!l_sg()
sub_0044A110:
lm_new.c!l_n36_buf() ;initialize
VENDORCODE structure and VENDORNAME
0044B503: push 00450BE0 ;push in lm_new.c!l_n36_buff() address
0044B508: call 00444B11 ;call lm_init.c!l_x77_buf() to set L_UNIQ_KEY5_FUNC
sub_00450BE0:
lm_new.c!l_n36_buff()
00450C34 – 00450EB0 ;obfuscate mem_ptr2_bytes[] in job
00450EB5 – 00450FF8 ;obfuscate data[0] and data[1] in VENDORCODE
sub_77F8DD80:
ntdll.dll!NtTerminateProcess()
77F8DD8B: ret 00000008 ;program exit
As we can see, the subroutine 0041A010 we mentioned
earlier is actually lm_ckout.c!lc_checkout(), indeed
a crucial function. It returns 0 if the license is successfully checked out,
otherwise it returns an error code (FFFFFFF8 is defined as LM_BADCODE in lmclient.h). l_checkout() may be
invoked several times due to heartbeat but we don’t care. Remember our aim is
to position the checksum comparison code and retrieve the real signature.
A quick search tells us that STRNCMP (a macro
defined in l_privat.h, sets result=0 if strings
match) appears only in lm_ckout.c!l_good_lic_key() and l_crypt.c!l_crypt_private(). Note l_crypt.c and l_strkey.c are not
directly compiled to objects, instead they are included in modules lm_ckout.obj and l_allfeat.obj. In
addition to those two, lm_crypt.obj also includes l_crypt.c and
exposes its functions to the outside as APIs, many under different aliases such
as lc_crypt(). However the copies in lm_crypt are not
used in cmath.exe: the address of lc_crypt() is
00469960, at which we set breakpoint, but nothing happened.
lm_ckout.c
... ...
#define
l_crypt_private l_ckout_crypt
#define
l_string_key l_ckout_string_key
... ...
/* Include l_crypt.c, so that these functions won’t be
global. */
#define LM_CKOUT
#include
"l_crypt.c"
l_allfeat.c
... ...
#define
LM_CRYPT_HASH
#include
"l_crypt.c"
l_crypt.c
#include
"l_strkey.c"
l_strkey.c
#include
"l_strkey.h"
lm_crypt.c
#define
l_crypt_private lc_crypt
... ...
#define LM_CRYPT
... ...
#include
"l_crypt.c"
Included more than once, the multiple copies of l_crypt/l_strkey
functions are not all the same due to compiling directives. The following l_crypt_private() code
illustrates that (STRNCMP here doesn’t really matter).
Depending on whether LM_CKOUT is defined, subroutines
0041EBE3 (long version) and 0043873B (short version) differ in the two modules.
IDA FLAIR recognizes the latter but not the former – in practice we have to
manually identify those functions in lm_ckout.obj.
ret = real_crypt(job, conf, sdate, code);
#ifdef LM_CKOUT
if (!(job->user_crypt_filter) &&
!(job->lc_this_keylist) && valid_code(conf->code))
{
if (job->flags
& LM_FLAG_MAKE_OLD_KEY)
{
STRNCMP(conf->code,
ret, MAX_CRYPT_LEN, not_eq);
}
else
{
STRNCMP(conf->lc_sign,
ret, MAX_CRYPT_LEN, not_eq);
}
if (not_eq
&& !(job->options->flags &
LM_OPTFLAG_STRINGS_CASE_SENSITIVE))
{
job->options->flags
|= LM_OPTFLAG_STRINGS_CASE_SENSITIVE;
ret =
real_crypt(job, conf, sdate, code);
job->options->flags
&= ~LM_OPTFLAG_STRINGS_CASE_SENSITIVE;
}
}
#endif /* LM_CKOUT */
return ret;
In the above code comments, we see that l_crypt_private() is the
key function that calculates the SIGN hash. There are two call chains involved,
initiated from lc_set_attr() when setting LM_A_LICENSE_FILE_PTR and lc_checkout()
respectively. The tracing result of l_crypt_private() was
21D5B6E8572E in the first case and 0 in the second. Clearly this looks like the
signature code and we immediately tried it in the license file. Unfortunately
it did not work.
Well, that’s not very surprising; the surprising thing is,
why would lc_set_attr() compute the checksum? Setting
attributes should have nothing to do with authentication. We must point out,
however, that during the call chain every line of license file is parsed,
literally! In our case there are 3 lines in license.dat but only
1 feature line, so l_parse_feature_line() is called three times but
oldkey() is
called only once. The most probable explanation, we think, is that it just
fills a hash code to the config structure as initialization, as
the name oldkey() suggests. Or, for conspiracy
theory fans, you can say this is a trick that lures crackers away from the real
thing to some camouflaged petty codes in order to waste their time.
OK, we are not that stupid, we know it is the checkout
call chain that counts. Because later half of the chain returns 0 rather than
the true hash, l_good_lic_key() reports failure (1=success,
0=failure), then lm_start_real() sets error number accordingly
and it’s relayed back to lc_checkout(). Note the STRNCMP code in l_good_lic_key() does not
behave as we expected. If the signature in license file is incorrect, then l_crypt_private() returns
0 and STRNCMP is bypassed; if it is correct, then l_crypt_private() returns
the same string and STRNCMP is totally meaningless. Either
way STRNCMP does not compare the wrong checksum with the right
one, as we speculated. Again you may have two perspectives on why FLEXlm did
this.
code = l_crypt_private(job, conf, sdate, &vc);
... ...
if (job->user_crypt_filter)
{
if (!code || !*code)
str_res = 1;
}
else
{
if
(conf->lc_keylist && job->L_SIGN_LEVEL)
{
if (!code ||
!*code || !*conf->code) /*P5552 */
str_res = 1;
else
STRNCMP(code, conf->lc_sign,
MAX_CRYPT_LEN, str_res);
}
else
{
if (!code ||
!*code || !*conf->code) /*P5552 */
str_res = 1;
else
STRNCMP(code, conf->code, MAX_CRYPT_LEN,
str_res);
}
}
if (str_res)
{
... ...
}
else
ok = 1;
Ignoring STRNCMP, we realized that we had to
trace all the way down to the bottom of the call chain to dig out the genuine
signature – it has to be computed and compared to license file input somewhere!
As it turns out, the place is l_string_key(). The user file signature
is passed in as argument, the correct license key is calculated, and then the
two are matched bit by bit. So this is where the right and wrong are revealed,
not those phony STRNCMPs.
#ifdef LM_CKOUT
static unsigned char *
l_string_key(job, input, inputlen, code, len, license_key)
#else
static unsigned char *
l_string_key(job, input, inputlen, code, len)
#endif
{
... ...
strcpy(lkey,
license_key);
... ... /* calculate y, the real
checksum */
#ifdef LM_CKOUT
for (i = 0; i < j; i++)
{
/* convert user checksum from ASCII to hex */
c = lkey[i * 2];
if (isdigit(c))
x = (c - '0')
<< 4;
else
x = ((c - 'A') +
10) << 4;
c = lkey[(i * 2) + 1];
if (isdigit(c))
x += (c - '0');
else
x += ((c - 'A')
+ 10);
/* compare user and real checksums */
if (x != y[i])
return 0;
}
... ...
#endif
ret = (atox(job, y, len));
return ret;
}
The rest of job is easy: we just trace in and grab y, the
correct license key. In reality we modified the matching result at runtime,
evading the “return 0;” branch, and let the function
return y in ASCII. Repeat the process for CSTAT, and we
obtain the following new license file.
SERVER hostname hostid
27000
DAEMON VNI
"<vni_dir>\license\bin\bin.i386nt\vni.exe"
FEATURE CMATH VNI 5.5
permanent uncounted 6D5C01FD71C9 HOSTID=ANY
FEATURE CSTAT VNI 5.5
permanent uncounted 369B56AC8B35 HOSTID=ANY
Test it with all validation scripts provided by VNI and
all pass with flying colors[7].
This accomplishment confirmed our hypothesis that every protection scheme has
to compare the right and the wrong in order to differentiate legitimate and
non-legitimate users. The key is to pinpoint that matching precisely in time
and space. To do that hackers must have some deep understandings of the code.
In our example we had some luxuries such as source code and LIB files that
commonly people cannot afford. I don’t know if I could make it without them.
Therefore solving license code is absolutely at higher level than patching.
[7] We tried
changing version to 6.5 or 7.0, but scripts did not pass. Evidently the SIGN
hash relies on version.
