FASM – A Simple Win32 Hex Editor | Assembly Language Programming

June 20, 2015

FASM – A Simple Win32 Hex Editor

Filed under: Assembly Language, C++ — Tags: Assembly Language, FASM, Win32, x86 Assembly Language — programmer209 @ 10:47 am

This post demonstrates how to implement a simple Win32 Hex Editor in x86 assembly language using FASM – the Flat Assembler.

The program displays the bytes from an input file as hexadecimal values. Hex values can also be edited or entered, and when saved will be converted back to byte values before being written to file.

FASM – A Simple Win32 Text Editor

FASM – Using The C File Functions

Opening A File

The program uses the C file functions to read and write files.

A file is opened for reading in binary mode by calling fopen. The size of the file is determined by first calling fseek to point to the end of the file, and then calling ftell to get the offset from the beginning of the file. Since the file has been opened in binary mode, this offset will be equal to the size of the file in bytes. The rewind function is then called to set the position back to the beginning of the file, so that it can be read.

The malloc function is then called to reserve memory for the buffer that will eventually be displayed in the Edit window. The buffer needs to be large enough to store 2 hex digit chars for each byte in the file, plus one space char for each 4 bytes in the file, plus a null terminator.

The file is read one byte at a time using the fgetc function. The byte is converted to 2 hex digits which are written to the buffer. Once all of the file bytes have been processed, SetWindowText is called to display the contents of the output buffer in the Edit window.

Finally, free is called to release the buffer, and fclose to close the input file.

Saving To File

When File|Save or File|Save As is selected, then whatever hexadecimal values are currently displayed in the Edit window will be converted back to byte values before being written to the output file.

The contents of the Edit window are copied to a buffer. The function GetWindowTextLength is called to get the size of the data in the Edit window. The buffer is allocated via malloc, and GetWindowText is called to fill the buffer.

Note that the Edit window can contain non hex digit characters, such as spaces and CRLFs. These will be ignored when the buffer is processed.

The output file is opened for writing in binary mode, by calling fopen. Then hex digits are read from the buffer 2 at a time and converted to a byte value. Each byte value is written to the output file by calling the fputc function.

Once the buffer has been processed, free is called to release the buffer, and fclose to close the output file.

The FASM Code

To grab this code, select with the mouse and then copy and paste directly into the FASM IDE.

; -------------------------------------------------------------------------------------

 format PE GUI 4.0

 entry win_main

 include 'win32a.inc'

; -------------------------------------------------------------------------------------

 IDR_MY_MENU = 1

 IDM_FILE = 100
 IDM_NEW = 101
 IDM_OPEN = 102
 IDM_SAVE = 103
 IDM_SAVE_AS = 104
 IDM_EXIT = 105

 IDR_ACCEL = 106
 ID_SELECT_ALL = 107

; -------------------------------------------------------------------------------------

 section '.code' code readable executable

  win_main:

	; initialise the members of the wcex structure

	; --------------------------------------------

	; WNDCLASSEX

	; UINT cbSize
	; UINT style
	; WNDPROC lpfnWndProc
	; int cbClsExtra
	; int cbWndExtra
	; HINSTANCE hInstance
	; HICON hIcon
	; HCURSOR hCursor
	; HBRUSH hbrBackground
	; LPCTSTR lpszMenuName
	; LPCTSTR lpszClassName
	; HICON hIconSm

	; --------------------------------------------

	; the instance handle

	invoke GetModuleHandle,0

	mov [wcex.hInstance],eax

	; cbSize

	mov eax,sizeof.WNDCLASSEX

	mov [wcex.cbSize],eax

	; the windows proc

	mov [wcex.lpfnWndProc],WndProc

	; the windows style

	mov [wcex.style],CS_HREDRAW+CS_VREDRAW

	; load the icons

	invoke LoadIcon,[wcex.hInstance],IDI_APPLICATION

	mov [wcex.hIcon],eax

	mov [wcex.hIconSm],eax

	; load the cursor

	invoke LoadCursor,NULL,IDC_ARROW

	mov [wcex.hCursor],eax

	; the brush for the background

	mov [wcex.hbrBackground],COLOR_BTNFACE+1

	; the windows class name

	mov dword [wcex.lpszClassName],szClass

	; set to NULL

	mov [wcex.cbClsExtra],0

	mov [wcex.cbWndExtra],0

	mov dword [wcex.lpszMenuName],NULL

	; register wcex

	; RegisterClassEx(&wcex)

	invoke RegisterClassEx,wcex

	test eax,eax

	jz reg_error

	; load the menu

	invoke LoadMenu,[wcex.hInstance],IDR_MY_MENU

	mov [h_menu],eax

	; create the main window

	invoke CreateWindowEx,0,szClass,szTitle,WS_OVERLAPPEDWINDOW,\
			      CW_USEDEFAULT,CW_USEDEFAULT,\
			      700,500,NULL,[h_menu],[wcex.hInstance],NULL

	; save the windows handle:

	mov [h_wnd],eax

	test eax,eax

	jz cr_error

	; load the accelerator table

	invoke LoadAccelerators,[wcex.hInstance],IDR_ACCEL

	test eax,eax

	jz acc_error

	mov [h_accel],eax

	; show and update the window

	; ShowWindow(hWnd,SW_SHOWNORMAL)

	invoke ShowWindow,[h_wnd],SW_SHOWNORMAL

	; UpdateWindow(hWnd)

	invoke UpdateWindow,[h_wnd]

  msg_loop:

	; the main message loop

	; GetMessage(&msg,NULL,0,0)

	invoke GetMessage,msg,NULL,0,0

	cmp eax,1

	jb exit

	jne msg_loop

	; TranslateAccelerator(hwnd,h_accel,&msg)

	invoke TranslateAccelerator,[h_wnd],[h_accel],msg

	test eax,eax

	; no need to call TranslateMessage and DispatchMessage,

	; if an accelerator is successfully translated

	jnz msg_loop

	; TranslateMessage(&msg)

	invoke TranslateMessage,msg

	; DispatchMessage(&msg)

	invoke DispatchMessage,msg

	jmp msg_loop

  reg_error:

	invoke MessageBox,NULL,szRegError,szTitle,MB_ICONERROR+MB_OK

	jmp exit

  cr_error:

	invoke MessageBox,NULL,szCreateError,szTitle,MB_ICONERROR+MB_OK

	jmp exit

  acc_error:

	invoke MessageBox,NULL,szAccelError,szTitle,MB_ICONERROR+MB_OK

  exit:

	; return msg.wParam

	invoke ExitProcess,[msg.wParam]

; -------------------------------------------------------------------------------------

proc WndProc uses ebx esi edi,hwnd,wmsg,wparam,lparam

  ; WndProc(hwnd,wmsg,wparam,lparam)

  ; callback function to process messages for the main window

	cmp [wmsg],WM_CREATE

	je .ON_CREATE

	cmp [wmsg],WM_SIZE

	je .SIZE

	cmp [wmsg],WM_SETFOCUS

	je .SET_FOCUS

	cmp [wmsg],WM_COMMAND

	je .COMMAND

	cmp [wmsg],WM_DESTROY

	je .DESTROY

  .DEFAULT:

	; DefWindowProc(hWnd,wmsg,wParam,lParam)

	invoke DefWindowProc,[hwnd],[wmsg],[wparam],[lparam]

	jmp .DONE

  .ON_CREATE:

	; create the child EDIT control

	invoke CreateWindowEx,WS_EX_CLIENTEDGE,szEdit,0,\
	WS_VISIBLE+WS_CHILD+WS_VSCROLL+ES_AUTOVSCROLL+ES_MULTILINE,\
	0,0,0,0,[hwnd],0,[wcex.hInstance],NULL

	test eax,eax

	jz .EDIT_FAILED

	mov [h_wnd_edit],eax

	; create a font for the edit control

	invoke CreateFont,18,0,0,0,FW_NORMAL,FALSE,FALSE,FALSE,ANSI_CHARSET,\
	OUT_DEVICE_PRECIS,CLIP_DEFAULT_PRECIS,PROOF_QUALITY,FIXED_PITCH+FF_DONTCARE,szFontFace

	test eax,eax

	jz .FONT_FAILED

	mov [h_font],eax

	invoke SendMessage,[h_wnd_edit],WM_SETFONT,[h_font],FALSE

	xor eax,eax

	jmp .DONE

  .EDIT_FAILED:

	invoke MessageBox,NULL,szCreateError,szTitle,MB_ICONERROR+MB_OK

	jmp .DONE

  .FONT_FAILED:

	invoke MessageBox,NULL,szFontError,szTitle,MB_ICONERROR+MB_OK

	jmp .DONE

  .SIZE:

	; resize the child window

	; new width = LOWORD(lParam)

	; new height = HIWORD(lParam)

	mov eax,[lparam]

	and eax,0xffff

	mov edx,[lparam]

	shr edx,16

	invoke MoveWindow,[h_wnd_edit],0,0,eax,edx,TRUE

	xor eax,eax

	jmp .DONE

  .SET_FOCUS:

	; set the focus to the child EDIT control

	invoke SetFocus,[h_wnd_edit]

	xor eax,eax

	jmp .DONE

  .COMMAND:

	mov eax,[wparam]

	and eax,0xffff

	cmp eax,IDM_NEW

	je .NEW

	cmp eax,IDM_OPEN

	je .OPEN

	cmp eax,IDM_SAVE

	je .SAVE

	cmp eax,IDM_SAVE_AS

	je .SAVE_AS

	cmp eax,IDM_EXIT

	je .DESTROY

	cmp eax,ID_SELECT_ALL

	je .SELECT_ALL

	jmp .DEFAULT

  .NEW:

	; File|New or CTRL+N

	; clear the Edit window

	invoke SendMessage,[h_wnd_edit],WM_SETTEXT,0,0

	invoke SetWindowText,[hwnd],szTitle

	; reset the file name

	lea esi,[szFileName]

	mov [esi],byte 0

	xor eax,eax

	jmp .DONE

  .OPEN:

	; File|Open or CTRL+O

	stdcall File_Open,[hwnd]

	test eax,eax

	jz .DONE

	; load the file contents into the Edit window

	stdcall File_To_Hex

	invoke SetWindowText,[hwnd],szFileName

	xor eax,eax

	jmp .DONE

  .SAVE:

	; File|Save or CTRL+S

	; if there isn't a valid file name,

	; call the File Save As dialog

	cmp [szFileName],byte 0

	jnz .HEX_TO_FILE

	stdcall File_Save_As,[hwnd]

	test eax,eax

	jz .DONE

	invoke SetWindowText,[hwnd],szFileName

  .HEX_TO_FILE:

	; save the Edit window data to file

	stdcall Hex_To_File

	xor eax,eax

	jmp .DONE

  .SAVE_AS:

	; File|Save As

	stdcall File_Save_As,[hwnd]

	test eax,eax

	jz .DONE

	invoke SetWindowText,[hwnd],szFileName

	; save the Edit window data to file

	stdcall Hex_To_File

	xor eax,eax

	jmp .DONE

  .SELECT_ALL:

	; CTRL+A (via Accerlerator Table)

	invoke SendMessage,[h_wnd_edit],EM_SETSEL,0,-1

	xor eax,eax

	jmp .DONE

  .DESTROY:

	; destroy the font

	invoke DeleteObject,[h_font]

	; PostQuitMessage(0)

	invoke PostQuitMessage,0

	xor eax,eax

  .DONE:

	ret

endp

; -------------------------------------------------------------------------------------

proc File_Open,hwnd

  ; display the GetOpenFileName dialog to get the file name

	; zero ofn - the OPENFILENAME struct

	mov eax,sizeof.OPENFILENAME

	cinvoke memset,ofn,0,eax

	; initialise the members of ofn

	mov eax,sizeof.OPENFILENAME

	mov [ofn.lStructSize],eax

	mov eax,[hwnd]

	mov [ofn.hwndOwner],eax

	; the filter

	mov eax,szFilter

	mov [ofn.lpstrFilter],eax

	lea eax,[szFileName]

	mov [ofn.lpstrFile],eax

	mov eax,MAX_PATH

	mov [ofn.nMaxFile],eax

	mov eax,OFN_EXPLORER
	or eax,OFN_FILEMUSTEXIST
	or eax,OFN_HIDEREADONLY

	mov [ofn.Flags],eax

	mov eax,szDefFileExtension

	mov [ofn.lpstrDefExt],eax

	; display the dialog to get szFileName

	invoke GetOpenFileName,ofn

	ret

endp

; -------------------------------------------------------------------------------------

proc File_Save_As,hwnd

  ; display the GetSaveFileName dialog to get the file name

	; zero ofn - the OPENFILENAME struct

	mov eax,sizeof.OPENFILENAME

	cinvoke memset,ofn,0,eax

	; initialise the members of ofn

	mov eax,sizeof.OPENFILENAME

	mov [ofn.lStructSize],eax

	mov eax,[hwnd]

	mov [ofn.hwndOwner],eax

	; the filter

	mov eax,szFilter

	mov [ofn.lpstrFilter],eax

	lea eax,[szFileName]

	mov [ofn.lpstrFile],eax

	mov eax,MAX_PATH

	mov [ofn.nMaxFile],eax

	mov eax,OFN_EXPLORER
	or eax,OFN_PATHMUSTEXIST
	or eax,OFN_HIDEREADONLY
	or eax,OFN_OVERWRITEPROMPT

	mov [ofn.Flags],eax

	mov eax,szDefFileExtension

	mov [ofn.lpstrDefExt],eax

	; display the dialog to get szFileName

	invoke GetSaveFileName,ofn

	ret

endp

; -------------------------------------------------------------------------------------

proc File_To_Hex

  ; read the file in binary mode

  ; convert each byte to a hex value

  ; display the hex data in the Edit window

	locals

	  fp dd 0 ; the file pointer

	  count dd 0

	endl

	; set the mode to "rb" - open for reading

	lea esi,[szMode]

	mov [esi],byte 'r'

	mov [esi+1],byte 'b'

	mov [esi+2],byte 0

	; open the file

	cinvoke fopen,szFileName,szMode

	mov [fp],eax

	test eax,eax

	jz .ERR_FOPEN

	; get the size of the file in bytes

	; 2 = SEEK_END

	cinvoke fseek,[fp],0,2

	cinvoke ftell,[fp]

	; compute the size of pEditBuffer

	mov edx,eax

	; 1 space char for each 4 bytes (8 hex chars)

	shr edx,2

	; 2 hex chars for each byte

	shl eax,1

	add eax,edx

	; +1 for the null terminator

	inc eax

	; allocate the buffer

	cinvoke malloc,eax

	test eax,eax

	jz .MALLOC_FAILED

	; point pEditBuffer to the buffer

	mov [pEditBuffer],eax

	; rewind the file

	cinvoke rewind,[fp]

	mov edi,[pEditBuffer]

  .LOOP:

	; read the file bytes

	cinvoke fgetc,[fp]

	; test for EOF

	cmp eax,0xffffffff

	je .FINISH

	stdcall Byte_To_Hex

	mov [edi],ax

	inc edi

	inc edi

	inc [count]

	; insert a space after every 4 bytes

	test [count],3

	jnz .LOOP

	mov [edi],byte 32

	inc edi

	jmp .LOOP

  .FINISH:

	; null terminate the buffer

	mov [edi],byte 0

	; set the text of the Edit window

	invoke SetWindowText,[h_wnd_edit],[pEditBuffer]

  .FREE:

	cinvoke free,[pEditBuffer]

  .FCLOSE:

	cinvoke fclose,[fp]

	jmp .DONE

  .MALLOC_FAILED:

	invoke MessageBox,NULL,szAllocError,szTitle,MB_ICONERROR+MB_OK

	jmp .FCLOSE

  .ERR_FOPEN:

	invoke MessageBox,NULL,szFileError,szTitle,MB_ICONERROR+MB_OK

  .DONE:

	ret

endp

; -------------------------------------------------------------------------------------

proc Hex_To_File

  ; write the contents of the Edit window to file

	; get the number of bytes in the Edit window

	invoke GetWindowTextLength,[h_wnd_edit]

	; +1 for the null terminator

	inc eax

	mov [nBytes],eax

	; allocate the buffer

	cinvoke malloc,eax

	test eax,eax

	jz .MALLOC_FAILED

	; point pEditBuffer to the buffer

	mov [pEditBuffer],eax

	; read the contents of the Edit window into the buffer

	invoke GetWindowText,[h_wnd_edit],[pEditBuffer],[nBytes]

	; remove all non hex digits from the buffer

	stdcall Filter_Non_Hex

	; the file pointer

	locals

	  fp dd 0

	endl

	; set the mode to "wb" - open for writing

	lea esi,[szMode]

	mov [esi],byte 'w'

	mov [esi+1],byte 'b'

	mov [esi+2],byte 0

	; open the file

	cinvoke fopen,szFileName,szMode

	mov [fp],eax

	test eax,eax

	jz .ERR_FOPEN

	; read the hex digits from the buffer

	mov esi,[pEditBuffer]

  .LOOP:

	; read 2 digits = 1 byte

	mov ax,[esi]

	cmp al,0

	je .FCLOSE

	cmp ah,0

	je .FCLOSE

	; convert to a byte value

	stdcall Digits_To_Byte

	and eax,0xff

	; write the byte to file

	cinvoke fputc,eax,[fp]

	inc esi

	inc esi

	jmp .LOOP

  .FCLOSE:

	cinvoke fclose,[fp]

  .FREE:

	cinvoke free,[pEditBuffer]

	jmp .DONE

  .ERR_FOPEN:

	invoke MessageBox,NULL,szFileError,szTitle,MB_ICONERROR+MB_OK

	jmp .FREE

  .MALLOC_FAILED:

	invoke MessageBox,NULL,szAllocError,szTitle,MB_ICONERROR+MB_OK

  .DONE:

	ret

endp

; -------------------------------------------------------------------------------------

proc Byte_To_Hex

  ; convert 1 byte to 2 hex digits

  ; byte passed in AL

  ; result returned in AX

	mov ah,al

	shr al,4

	and ah,0xf

	cmp al,9

	jg .AF0

	add al,'0'

	jmp .AH

  .AF0:

	sub al,10

	add al,'A'

  .AH:

	cmp ah,9

	jg .AF1

	add ah,'0'

	jmp .DONE

  .AF1:

	sub ah,10

	add ah,'A'

  .DONE:

	ret

endp

; -------------------------------------------------------------------------------------

proc Digits_To_Byte

  ; convert 2 hex digits to a byte value

  ; digits passed in AX

  ; result returned in AL

	sub al,'0'

	test al,16

	jz .NEXT

	sub al,7

  .NEXT:

	sub ah,'0'

	test ah,16

	jz .BYTE

	sub ah,7

  .BYTE:

	shl al,4

	or al,ah

	ret

endp

; -------------------------------------------------------------------------------------

proc Filter_Non_Hex

  ; remove all non hex chars from the buffer

	mov esi,[pEditBuffer]

	mov edi,[pEditBuffer]

  .SCAN:

	mov al,[esi]

	cmp al,0

	je .DONE

	cmp al,'0'

	jl .NEXT

	cmp al,'9'

	jg .AF

	jmp .COPY

  .AF:

	cmp al,'A'

	jl .NEXT

	cmp al,'F'

	jg .af

	jmp .COPY

  .af:

	cmp al,'a'

	jl .NEXT

	cmp al,'f'

	jg .NEXT

	; convert to upper case

	sub al,'a'

	add al,'A'

  .COPY:

	mov [edi],al

	inc edi

  .NEXT:

	inc esi

	jmp .SCAN

  .DONE:

	; null terminate

	mov [edi],byte 0

	ret

endp

; -------------------------------------------------------------------------------------

section '.idata' import data readable writeable

  library kernel,'KERNEL32.DLL',\
	  user,'USER32.DLL',\
	  comdlg32,'COMDLG32.DLL',\
	  gdi,'GDI32.DLL',\
	  msvcrt,'MSVCRT.DLL'

  import kernel,\
	 GetModuleHandle,'GetModuleHandleA',\
	 ExitProcess,'ExitProcess'

  import user,\
	 RegisterClassEx,'RegisterClassExA',\
	 CreateWindowEx,'CreateWindowExA',\
	 ShowWindow,'ShowWindow',\
	 UpdateWindow,'UpdateWindow',\
	 MoveWindow,'MoveWindow',\
	 GetMessage,'GetMessageA',\
	 SendMessage,'SendMessageA',\
	 TranslateMessage,'TranslateMessage',\
	 DispatchMessage,'DispatchMessageA',\
	 MessageBox,'MessageBoxA',\
	 DefWindowProc,'DefWindowProcA',\
	 PostQuitMessage,'PostQuitMessage',\
	 SetFocus,'SetFocus',\
	 LoadIcon,'LoadIconA',\
	 LoadMenu,'LoadMenuA',\
	 LoadCursor,'LoadCursorA',\
	 SetWindowText,'SetWindowTextA',\
	 GetWindowText,'GetWindowTextA',\
	 GetWindowTextLength,'GetWindowTextLengthA',\
	 LoadAccelerators,'LoadAcceleratorsA',\
	 TranslateAccelerator,'TranslateAcceleratorA'

  import comdlg32,\
	 GetOpenFileName,'GetOpenFileNameA',\
	 GetSaveFileName,'GetSaveFileNameA'

  import gdi,\
	 CreateFont,'CreateFontA',\
	 DeleteObject,'DeleteObject'

  import msvcrt,\
	 memset,'memset',\
	 malloc,'malloc',\
	 free,'free',\
	 fopen,'fopen',\
	 fclose,'fclose',\
	 fgetc,'fgetc',\
	 fputc,'fputc',\
	 ftell,'ftell',\
	 fseek,'fseek',\
	 rewind,'rewind'

; -------------------------------------------------------------------------------------

section '.data' readable writeable

  szClass TCHAR "Win32app",0

  szTitle TCHAR "Hex Editor",0

  szEdit TCHAR "EDIT",0

  szMode rb 4

  szRegError TCHAR "Call to RegisterClassEx failed!",0

  szCreateError TCHAR "Call to CreateWindowEx failed!",0

  szFontError TCHAR "Call to CreateFont failed!",0

  szFileError TCHAR "Call to fopen() failed!",0

  szAllocError TCHAR "Call to malloc() failed!",0

  szAccelError TCHAR "Call to LoadAccelerators failed!",0

  szFontFace TCHAR "Lucida Console",0

  szDefFileExtension TCHAR "txt",0

  ; Note that the filter string is double null terminated:

  szFilter TCHAR "Text Files",0,"*.txt",0,"All Files",0,"*.*",0,0

  szFileName rb MAX_PATH

  ofn OPENFILENAME

  wcex WNDCLASSEX

  msg MSG

  h_wnd dd 0

  h_wnd_edit dd 0

  h_font dd 0

  h_menu dd 0

  h_accel dd 0

  nBytes dd 0

  pEditBuffer dd 0

; -------------------------------------------------------------------------------------

section '.rc' resource data readable

  directory RT_MENU,menus,\
	    RT_ACCELERATOR,accelerators

  resource menus,IDR_MY_MENU,LANG_ENGLISH+SUBLANG_DEFAULT,my_menu

  resource accelerators,IDR_ACCEL,LANG_ENGLISH+SUBLANG_DEFAULT,accel_keys

  menu my_menu

  menuitem 'File',IDM_FILE,MFR_POPUP+MFR_END
  menuitem 'New',IDM_NEW
  menuitem 'Open',IDM_OPEN
  menuitem 'Save',IDM_SAVE
  menuitem 'Save As',IDM_SAVE_AS
  menuseparator
  menuitem 'Exit',IDM_EXIT,MFR_END

  accelerator accel_keys,\
	      FVIRTKEY+FNOINVERT+FCONTROL,'N',IDM_NEW,\
	      FVIRTKEY+FNOINVERT+FCONTROL,'O',IDM_OPEN,\
	      FVIRTKEY+FNOINVERT+FCONTROL,'S',IDM_SAVE,\
	      FVIRTKEY+FNOINVERT+FCONTROL,'A',ID_SELECT_ALL

; -------------------------------------------------------------------------------------

Assembly Language Programming

June 20, 2015