FASM – A Simple Win32 Hex Editor

This post demonstrates how to implement a simple Win32 Hex Editor in x86 assembly language using FASM – the Flat Assembler.

The program displays the bytes from an input file as hexadecimal values. Hex values can also be edited or entered, and when saved will be converted back to byte values before being written to file.

Related Posts

FASM – A Simple Win32 Text Editor

FASM – Using The C File Functions

Opening A File

The program uses the C file functions to read and write files.

A file is opened for reading in binary mode by calling fopen. The size of the file is determined by first calling fseek to point to the end of the file, and then calling ftell to get the offset from the beginning of the file. Since the file has been opened in binary mode, this offset will be equal to the size of the file in bytes. The rewind function is then called to set the position back to the beginning of the file, so that it can be read.

The malloc function is then called to reserve memory for the buffer that will eventually be displayed in the Edit window. The buffer needs to be large enough to store 2 hex digit chars for each byte in the file, plus one space char for each 4 bytes in the file, plus a null terminator.

The file is read one byte at a time using the fgetc function. The byte is converted to 2 hex digits which are written to the buffer. Once all of the file bytes have been processed, SetWindowText is called to display the contents of the output buffer in the Edit window.

Finally, free is called to release the buffer, and fclose to close the input file.

Saving To File

When File|Save or File|Save As is selected, then whatever hexadecimal values are currently displayed in the Edit window will be converted back to byte values before being written to the output file.

The contents of the Edit window are copied to a buffer. The function GetWindowTextLength is called to get the size of the data in the Edit window. The buffer is allocated via malloc, and GetWindowText is called to fill the buffer.

Note that the Edit window can contain non hex digit characters, such as spaces and CRLFs. These will be ignored when the buffer is processed.

The output file is opened for writing in binary mode, by calling fopen. Then hex digits are read from the buffer 2 at a time and converted to a byte value. Each byte value is written to the output file by calling the fputc function.

Once the buffer has been processed, free is called to release the buffer, and fclose to close the output file.

The FASM Code

To grab this code, select with the mouse and then copy and paste directly into the FASM IDE.

; ------------------------------------------------------------------------------------- format PE GUI 4.0 entry win_main include 'win32a.inc' ; ------------------------------------------------------------------------------------- IDR_MY_MENU = 1 IDM_FILE = 100 IDM_NEW = 101 IDM_OPEN = 102 IDM_SAVE = 103 IDM_SAVE_AS = 104 IDM_EXIT = 105 IDR_ACCEL = 106 ID_SELECT_ALL = 107 ; ------------------------------------------------------------------------------------- section '.code' code readable executable win_main: ; initialise the members of the wcex structure ; -------------------------------------------- ; WNDCLASSEX ; UINT cbSize ; UINT style ; WNDPROC lpfnWndProc ; int cbClsExtra ; int cbWndExtra ; HINSTANCE hInstance ; HICON hIcon ; HCURSOR hCursor ; HBRUSH hbrBackground ; LPCTSTR lpszMenuName ; LPCTSTR lpszClassName ; HICON hIconSm ; -------------------------------------------- ; the instance handle invoke GetModuleHandle,0 mov [wcex.hInstance],eax ; cbSize mov eax,sizeof.WNDCLASSEX mov [wcex.cbSize],eax ; the windows proc mov [wcex.lpfnWndProc],WndProc ; the windows style mov [wcex.style],CS_HREDRAW+CS_VREDRAW ; load the icons invoke LoadIcon,[wcex.hInstance],IDI_APPLICATION mov [wcex.hIcon],eax mov [wcex.hIconSm],eax ; load the cursor invoke LoadCursor,NULL,IDC_ARROW mov [wcex.hCursor],eax ; the brush for the background mov [wcex.hbrBackground],COLOR_BTNFACE+1 ; the windows class name mov dword [wcex.lpszClassName],szClass ; set to NULL mov [wcex.cbClsExtra],0 mov [wcex.cbWndExtra],0 mov dword [wcex.lpszMenuName],NULL ; register wcex ; RegisterClassEx(&wcex) invoke RegisterClassEx,wcex test eax,eax jz reg_error ; load the menu invoke LoadMenu,[wcex.hInstance],IDR_MY_MENU mov [h_menu],eax ; create the main window invoke CreateWindowEx,0,szClass,szTitle,WS_OVERLAPPEDWINDOW,\ CW_USEDEFAULT,CW_USEDEFAULT,\ 700,500,NULL,[h_menu],[wcex.hInstance],NULL ; save the windows handle: mov [h_wnd],eax test eax,eax jz cr_error ; load the accelerator table invoke LoadAccelerators,[wcex.hInstance],IDR_ACCEL test eax,eax jz acc_error mov [h_accel],eax ; show and update the window ; ShowWindow(hWnd,SW_SHOWNORMAL) invoke ShowWindow,[h_wnd],SW_SHOWNORMAL ; UpdateWindow(hWnd) invoke UpdateWindow,[h_wnd] msg_loop: ; the main message loop ; GetMessage(&msg,NULL,0,0) invoke GetMessage,msg,NULL,0,0 cmp eax,1 jb exit jne msg_loop ; TranslateAccelerator(hwnd,h_accel,&msg) invoke TranslateAccelerator,[h_wnd],[h_accel],msg test eax,eax ; no need to call TranslateMessage and DispatchMessage, ; if an accelerator is successfully translated jnz msg_loop ; TranslateMessage(&msg) invoke TranslateMessage,msg ; DispatchMessage(&msg) invoke DispatchMessage,msg jmp msg_loop reg_error: invoke MessageBox,NULL,szRegError,szTitle,MB_ICONERROR+MB_OK jmp exit cr_error: invoke MessageBox,NULL,szCreateError,szTitle,MB_ICONERROR+MB_OK jmp exit acc_error: invoke MessageBox,NULL,szAccelError,szTitle,MB_ICONERROR+MB_OK exit: ; return msg.wParam invoke ExitProcess,[msg.wParam] ; ------------------------------------------------------------------------------------- proc WndProc uses ebx esi edi,hwnd,wmsg,wparam,lparam ; WndProc(hwnd,wmsg,wparam,lparam) ; callback function to process messages for the main window cmp [wmsg],WM_CREATE je .ON_CREATE cmp [wmsg],WM_SIZE je .SIZE cmp [wmsg],WM_SETFOCUS je .SET_FOCUS cmp [wmsg],WM_COMMAND je .COMMAND cmp [wmsg],WM_DESTROY je .DESTROY .DEFAULT: ; DefWindowProc(hWnd,wmsg,wParam,lParam) invoke DefWindowProc,[hwnd],[wmsg],[wparam],[lparam] jmp .DONE .ON_CREATE: ; create the child EDIT control invoke CreateWindowEx,WS_EX_CLIENTEDGE,szEdit,0,\ WS_VISIBLE+WS_CHILD+WS_VSCROLL+ES_AUTOVSCROLL+ES_MULTILINE,\ 0,0,0,0,[hwnd],0,[wcex.hInstance],NULL test eax,eax jz .EDIT_FAILED mov [h_wnd_edit],eax ; create a font for the edit control invoke CreateFont,18,0,0,0,FW_NORMAL,FALSE,FALSE,FALSE,ANSI_CHARSET,\ OUT_DEVICE_PRECIS,CLIP_DEFAULT_PRECIS,PROOF_QUALITY,FIXED_PITCH+FF_DONTCARE,szFontFace test eax,eax jz .FONT_FAILED mov [h_font],eax invoke SendMessage,[h_wnd_edit],WM_SETFONT,[h_font],FALSE xor eax,eax jmp .DONE .EDIT_FAILED: invoke MessageBox,NULL,szCreateError,szTitle,MB_ICONERROR+MB_OK jmp .DONE .FONT_FAILED: invoke MessageBox,NULL,szFontError,szTitle,MB_ICONERROR+MB_OK jmp .DONE .SIZE: ; resize the child window ; new width = LOWORD(lParam) ; new height = HIWORD(lParam) mov eax,[lparam] and eax,0xffff mov edx,[lparam] shr edx,16 invoke MoveWindow,[h_wnd_edit],0,0,eax,edx,TRUE xor eax,eax jmp .DONE .SET_FOCUS: ; set the focus to the child EDIT control invoke SetFocus,[h_wnd_edit] xor eax,eax jmp .DONE .COMMAND: mov eax,[wparam] and eax,0xffff cmp eax,IDM_NEW je .NEW cmp eax,IDM_OPEN je .OPEN cmp eax,IDM_SAVE je .SAVE cmp eax,IDM_SAVE_AS je .SAVE_AS cmp eax,IDM_EXIT je .DESTROY cmp eax,ID_SELECT_ALL je .SELECT_ALL jmp .DEFAULT .NEW: ; File|New or CTRL+N ; clear the Edit window invoke SendMessage,[h_wnd_edit],WM_SETTEXT,0,0 invoke SetWindowText,[hwnd],szTitle ; reset the file name lea esi,[szFileName] mov [esi],byte 0 xor eax,eax jmp .DONE .OPEN: ; File|Open or CTRL+O stdcall File_Open,[hwnd] test eax,eax jz .DONE ; load the file contents into the Edit window stdcall File_To_Hex invoke SetWindowText,[hwnd],szFileName xor eax,eax jmp .DONE .SAVE: ; File|Save or CTRL+S ; if there isn't a valid file name, ; call the File Save As dialog cmp [szFileName],byte 0 jnz .HEX_TO_FILE stdcall File_Save_As,[hwnd] test eax,eax jz .DONE invoke SetWindowText,[hwnd],szFileName .HEX_TO_FILE: ; save the Edit window data to file stdcall Hex_To_File xor eax,eax jmp .DONE .SAVE_AS: ; File|Save As stdcall File_Save_As,[hwnd] test eax,eax jz .DONE invoke SetWindowText,[hwnd],szFileName ; save the Edit window data to file stdcall Hex_To_File xor eax,eax jmp .DONE .SELECT_ALL: ; CTRL+A (via Accerlerator Table) invoke SendMessage,[h_wnd_edit],EM_SETSEL,0,-1 xor eax,eax jmp .DONE .DESTROY: ; destroy the font invoke DeleteObject,[h_font] ; PostQuitMessage(0) invoke PostQuitMessage,0 xor eax,eax .DONE: ret endp ; ------------------------------------------------------------------------------------- proc File_Open,hwnd ; display the GetOpenFileName dialog to get the file name ; zero ofn - the OPENFILENAME struct mov eax,sizeof.OPENFILENAME cinvoke memset,ofn,0,eax ; initialise the members of ofn mov eax,sizeof.OPENFILENAME mov [ofn.lStructSize],eax mov eax,[hwnd] mov [ofn.hwndOwner],eax ; the filter mov eax,szFilter mov [ofn.lpstrFilter],eax lea eax,[szFileName] mov [ofn.lpstrFile],eax mov eax,MAX_PATH mov [ofn.nMaxFile],eax mov eax,OFN_EXPLORER or eax,OFN_FILEMUSTEXIST or eax,OFN_HIDEREADONLY mov [ofn.Flags],eax mov eax,szDefFileExtension mov [ofn.lpstrDefExt],eax ; display the dialog to get szFileName invoke GetOpenFileName,ofn ret endp ; ------------------------------------------------------------------------------------- proc File_Save_As,hwnd ; display the GetSaveFileName dialog to get the file name ; zero ofn - the OPENFILENAME struct mov eax,sizeof.OPENFILENAME cinvoke memset,ofn,0,eax ; initialise the members of ofn mov eax,sizeof.OPENFILENAME mov [ofn.lStructSize],eax mov eax,[hwnd] mov [ofn.hwndOwner],eax ; the filter mov eax,szFilter mov [ofn.lpstrFilter],eax lea eax,[szFileName] mov [ofn.lpstrFile],eax mov eax,MAX_PATH mov [ofn.nMaxFile],eax mov eax,OFN_EXPLORER or eax,OFN_PATHMUSTEXIST or eax,OFN_HIDEREADONLY or eax,OFN_OVERWRITEPROMPT mov [ofn.Flags],eax mov eax,szDefFileExtension mov [ofn.lpstrDefExt],eax ; display the dialog to get szFileName invoke GetSaveFileName,ofn ret endp ; ------------------------------------------------------------------------------------- proc File_To_Hex ; read the file in binary mode ; convert each byte to a hex value ; display the hex data in the Edit window locals fp dd 0 ; the file pointer count dd 0 endl ; set the mode to "rb" - open for reading lea esi,[szMode] mov [esi],byte 'r' mov [esi+1],byte 'b' mov [esi+2],byte 0 ; open the file cinvoke fopen,szFileName,szMode mov [fp],eax test eax,eax jz .ERR_FOPEN ; get the size of the file in bytes ; 2 = SEEK_END cinvoke fseek,[fp],0,2 cinvoke ftell,[fp] ; compute the size of pEditBuffer mov edx,eax ; 1 space char for each 4 bytes (8 hex chars) shr edx,2 ; 2 hex chars for each byte shl eax,1 add eax,edx ; +1 for the null terminator inc eax ; allocate the buffer cinvoke malloc,eax test eax,eax jz .MALLOC_FAILED ; point pEditBuffer to the buffer mov [pEditBuffer],eax ; rewind the file cinvoke rewind,[fp] mov edi,[pEditBuffer] .LOOP: ; read the file bytes cinvoke fgetc,[fp] ; test for EOF cmp eax,0xffffffff je .FINISH stdcall Byte_To_Hex mov [edi],ax inc edi inc edi inc [count] ; insert a space after every 4 bytes test [count],3 jnz .LOOP mov [edi],byte 32 inc edi jmp .LOOP .FINISH: ; null terminate the buffer mov [edi],byte 0 ; set the text of the Edit window invoke SetWindowText,[h_wnd_edit],[pEditBuffer] .FREE: cinvoke free,[pEditBuffer] .FCLOSE: cinvoke fclose,[fp] jmp .DONE .MALLOC_FAILED: invoke MessageBox,NULL,szAllocError,szTitle,MB_ICONERROR+MB_OK jmp .FCLOSE .ERR_FOPEN: invoke MessageBox,NULL,szFileError,szTitle,MB_ICONERROR+MB_OK .DONE: ret endp ; ------------------------------------------------------------------------------------- proc Hex_To_File ; write the contents of the Edit window to file ; get the number of bytes in the Edit window invoke GetWindowTextLength,[h_wnd_edit] ; +1 for the null terminator inc eax mov [nBytes],eax ; allocate the buffer cinvoke malloc,eax test eax,eax jz .MALLOC_FAILED ; point pEditBuffer to the buffer mov [pEditBuffer],eax ; read the contents of the Edit window into the buffer invoke GetWindowText,[h_wnd_edit],[pEditBuffer],[nBytes] ; remove all non hex digits from the buffer stdcall Filter_Non_Hex ; the file pointer locals fp dd 0 endl ; set the mode to "wb" - open for writing lea esi,[szMode] mov [esi],byte 'w' mov [esi+1],byte 'b' mov [esi+2],byte 0 ; open the file cinvoke fopen,szFileName,szMode mov [fp],eax test eax,eax jz .ERR_FOPEN ; read the hex digits from the buffer mov esi,[pEditBuffer] .LOOP: ; read 2 digits = 1 byte mov ax,[esi] cmp al,0 je .FCLOSE cmp ah,0 je .FCLOSE ; convert to a byte value stdcall Digits_To_Byte and eax,0xff ; write the byte to file cinvoke fputc,eax,[fp] inc esi inc esi jmp .LOOP .FCLOSE: cinvoke fclose,[fp] .FREE: cinvoke free,[pEditBuffer] jmp .DONE .ERR_FOPEN: invoke MessageBox,NULL,szFileError,szTitle,MB_ICONERROR+MB_OK jmp .FREE .MALLOC_FAILED: invoke MessageBox,NULL,szAllocError,szTitle,MB_ICONERROR+MB_OK .DONE: ret endp ; ------------------------------------------------------------------------------------- proc Byte_To_Hex ; convert 1 byte to 2 hex digits ; byte passed in AL ; result returned in AX mov ah,al shr al,4 and ah,0xf cmp al,9 jg .AF0 add al,'0' jmp .AH .AF0: sub al,10 add al,'A' .AH: cmp ah,9 jg .AF1 add ah,'0' jmp .DONE .AF1: sub ah,10 add ah,'A' .DONE: ret endp ; ------------------------------------------------------------------------------------- proc Digits_To_Byte ; convert 2 hex digits to a byte value ; digits passed in AX ; result returned in AL sub al,'0' test al,16 jz .NEXT sub al,7 .NEXT: sub ah,'0' test ah,16 jz .BYTE sub ah,7 .BYTE: shl al,4 or al,ah ret endp ; ------------------------------------------------------------------------------------- proc Filter_Non_Hex ; remove all non hex chars from the buffer mov esi,[pEditBuffer] mov edi,[pEditBuffer] .SCAN: mov al,[esi] cmp al,0 je .DONE cmp al,'0' jl .NEXT cmp al,'9' jg .AF jmp .COPY .AF: cmp al,'A' jl .NEXT cmp al,'F' jg .af jmp .COPY .af: cmp al,'a' jl .NEXT cmp al,'f' jg .NEXT ; convert to upper case sub al,'a' add al,'A' .COPY: mov [edi],al inc edi .NEXT: inc esi jmp .SCAN .DONE: ; null terminate mov [edi],byte 0 ret endp ; ------------------------------------------------------------------------------------- section '.idata' import data readable writeable library kernel,'KERNEL32.DLL',\ user,'USER32.DLL',\ comdlg32,'COMDLG32.DLL',\ gdi,'GDI32.DLL',\ msvcrt,'MSVCRT.DLL' import kernel,\ GetModuleHandle,'GetModuleHandleA',\ ExitProcess,'ExitProcess' import user,\ RegisterClassEx,'RegisterClassExA',\ CreateWindowEx,'CreateWindowExA',\ ShowWindow,'ShowWindow',\ UpdateWindow,'UpdateWindow',\ MoveWindow,'MoveWindow',\ GetMessage,'GetMessageA',\ SendMessage,'SendMessageA',\ TranslateMessage,'TranslateMessage',\ DispatchMessage,'DispatchMessageA',\ MessageBox,'MessageBoxA',\ DefWindowProc,'DefWindowProcA',\ PostQuitMessage,'PostQuitMessage',\ SetFocus,'SetFocus',\ LoadIcon,'LoadIconA',\ LoadMenu,'LoadMenuA',\ LoadCursor,'LoadCursorA',\ SetWindowText,'SetWindowTextA',\ GetWindowText,'GetWindowTextA',\ GetWindowTextLength,'GetWindowTextLengthA',\ LoadAccelerators,'LoadAcceleratorsA',\ TranslateAccelerator,'TranslateAcceleratorA' import comdlg32,\ GetOpenFileName,'GetOpenFileNameA',\ GetSaveFileName,'GetSaveFileNameA' import gdi,\ CreateFont,'CreateFontA',\ DeleteObject,'DeleteObject' import msvcrt,\ memset,'memset',\ malloc,'malloc',\ free,'free',\ fopen,'fopen',\ fclose,'fclose',\ fgetc,'fgetc',\ fputc,'fputc',\ ftell,'ftell',\ fseek,'fseek',\ rewind,'rewind' ; ------------------------------------------------------------------------------------- section '.data' readable writeable szClass TCHAR "Win32app",0 szTitle TCHAR "Hex Editor",0 szEdit TCHAR "EDIT",0 szMode rb 4 szRegError TCHAR "Call to RegisterClassEx failed!",0 szCreateError TCHAR "Call to CreateWindowEx failed!",0 szFontError TCHAR "Call to CreateFont failed!",0 szFileError TCHAR "Call to fopen() failed!",0 szAllocError TCHAR "Call to malloc() failed!",0 szAccelError TCHAR "Call to LoadAccelerators failed!",0 szFontFace TCHAR "Lucida Console",0 szDefFileExtension TCHAR "txt",0 ; Note that the filter string is double null terminated: szFilter TCHAR "Text Files",0,"*.txt",0,"All Files",0,"*.*",0,0 szFileName rb MAX_PATH ofn OPENFILENAME wcex WNDCLASSEX msg MSG h_wnd dd 0 h_wnd_edit dd 0 h_font dd 0 h_menu dd 0 h_accel dd 0 nBytes dd 0 pEditBuffer dd 0 ; ------------------------------------------------------------------------------------- section '.rc' resource data readable directory RT_MENU,menus,\ RT_ACCELERATOR,accelerators resource menus,IDR_MY_MENU,LANG_ENGLISH+SUBLANG_DEFAULT,my_menu resource accelerators,IDR_ACCEL,LANG_ENGLISH+SUBLANG_DEFAULT,accel_keys menu my_menu menuitem 'File',IDM_FILE,MFR_POPUP+MFR_END menuitem 'New',IDM_NEW menuitem 'Open',IDM_OPEN menuitem 'Save',IDM_SAVE menuitem 'Save As',IDM_SAVE_AS menuseparator menuitem 'Exit',IDM_EXIT,MFR_END accelerator accel_keys,\ FVIRTKEY+FNOINVERT+FCONTROL,'N',IDM_NEW,\ FVIRTKEY+FNOINVERT+FCONTROL,'O',IDM_OPEN,\ FVIRTKEY+FNOINVERT+FCONTROL,'S',IDM_SAVE,\ FVIRTKEY+FNOINVERT+FCONTROL,'A',ID_SELECT_ALL ; -------------------------------------------------------------------------------------