windows: fix ShellExecute return condition

ShellExecute is an ancient API with an unusual return signature. It
pretends to return an HINSTANCE for backwards compatibility with Windows
3, but it's actualy a fake HINSTANCE. What's really happening here,
according to MSDN, is that it returns either an Windows error less
than or equal to 32, or it succeeds. So we adjust the return value
accordingly.

Prior to this commit, it was impossible to tell whether this command had
succeeded. For example, when using the "runas" verb, ShellExecute did
not correctly indicate whether or not permission was granted.

Change-Id: Ie60554d6465798bacb9a225c4ead7e8dd62bce14
Reviewed-on: https://go-review.googlesource.com/c/sys/+/199521
Run-TryBot: Jason A. Donenfeld <Jason@zx2c4.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nenad Kozul <knenad@gmail.com>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>

windows/syscall_windows.go

@@ -181,7 +181,7 @@ func NewCallbackCDecl(fn interface{}) uintptr {
 //sys	CancelIoEx(s Handle, o *Overlapped) (err error)
 //sys	CreateProcess(appName *uint16, commandLine *uint16, procSecurity *SecurityAttributes, threadSecurity *SecurityAttributes, inheritHandles bool, creationFlags uint32, env *uint16, currentDir *uint16, startupInfo *StartupInfo, outProcInfo *ProcessInformation) (err error) = CreateProcessW
 //sys	OpenProcess(desiredAccess uint32, inheritHandle bool, processId uint32) (handle Handle, err error)
-//sys	ShellExecute(hwnd Handle, verb *uint16, file *uint16, args *uint16, cwd *uint16, showCmd int32) (err error) = shell32.ShellExecuteW
+//sys	ShellExecute(hwnd Handle, verb *uint16, file *uint16, args *uint16, cwd *uint16, showCmd int32) (err error) [failretval<=32] = shell32.ShellExecuteW
 //sys	shGetKnownFolderPath(id *KNOWNFOLDERID, flags uint32, token Token, path **uint16) (ret error) = shell32.SHGetKnownFolderPath
 //sys	TerminateProcess(handle Handle, exitcode uint32) (err error)
 //sys	GetExitCodeProcess(handle Handle, exitcode *uint32) (err error)

windows/zsyscall_windows.go

@@ -1206,7 +1206,7 @@ func OpenProcess(desiredAccess uint32, inheritHandle bool, processId uint32) (ha
 
 func ShellExecute(hwnd Handle, verb *uint16, file *uint16, args *uint16, cwd *uint16, showCmd int32) (err error) {
 	r1, _, e1 := syscall.Syscall6(procShellExecuteW.Addr(), 6, uintptr(hwnd), uintptr(unsafe.Pointer(verb)), uintptr(unsafe.Pointer(file)), uintptr(unsafe.Pointer(args)), uintptr(unsafe.Pointer(cwd)), uintptr(showCmd))
-	if r1 == 0 {
+	if r1 <= 32 {
 		if e1 != 0 {
 			err = errnoErr(e1)
 		} else {