build: deal with endian mess

There is no standard mechanism for detecting endianess. Doing it at
compile time in a portable way is probably hard. Doing it properly
with a configure check is probably hard too. Using the endian
definitions in <sys/types.h> (usually includes <endian.h>, which is
not available everywhere) works under circumstances, but the previous
commit broke it on OSX.

Ideally all code should be endian dependent, but that is not possible
due to the dependencies (such as FFmpeg, some video output APIs, some
audio output APIs).

Create a header osdep/endian.h, which contains various fallbacks.
Note that the last fallback uses libavutil; however, it's not clear
whether AV_HAVE_BIGENDIAN is a public symbol, or whether including
<libavutil/bswap.h> really makes it visible. And in fact we don't want
to pollute the namespace with libavutil definitions either. Thus it's
only the last fallback.

audio/format.h

@@ -24,8 +24,8 @@
 #define MPLAYER_AF_FORMAT_H
 
 #include <stdbool.h>
-#include <sys/types.h>
 
+#include "osdep/endian.h"
 #include "bstr/bstr.h"
 
 #if BYTE_ORDER == BIG_ENDIAN

osdep/endian.h

@@ -0,0 +1,31 @@
+#ifndef MP_ENDIAN_H_
+#define MP_ENDIAN_H_
+
+#include <sys/types.h>
+
+#if !defined(BYTE_ORDER)
+
+#if defined(__BYTE_ORDER)
+#define BYTE_ORDER      __BYTE_ORDER
+#define LITTLE_ENDIAN   __LITTLE_ENDIAN
+#define BIG_ENDIAN      __BIG_ENDIAN
+#elif defined(__DARWIN_BYTE_ORDER)
+#define BYTE_ORDER      __DARWIN_BYTE_ORDER
+#define LITTLE_ENDIAN   __DARWIN_LITTLE_ENDIAN
+#define BIG_ENDIAN      __DARWIN_BIG_ENDIAN
+#else
+#include <libavutil/bswap.h>
+#if AV_HAVE_BIGENDIAN
+#define BYTE_ORDER      1234
+#define LITTLE_ENDIAN   4321
+#define BIG_ENDIAN      1234
+#else
+#define BYTE_ORDER      1234
+#define LITTLE_ENDIAN   1234
+#define BIG_ENDIAN      4321
+#endif
+#endif
+
+#endif /* !defined(BYTE_ORDER) */
+
+#endif

stream/stream_cdda.c

@@ -37,7 +37,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdbool.h>
-#include <sys/types.h>
+
+#include "osdep/endian.h"
 
 #include "talloc.h"
 

video/img_format.h

@@ -20,7 +20,8 @@
 #define MPLAYER_IMG_FORMAT_H
 
 #include <inttypes.h>
-#include <sys/types.h>
+
+#include "osdep/endian.h"
 #include "bstr/bstr.h"
 
 #if BYTE_ORDER == BIG_ENDIAN

video/img_fourcc.h

@@ -1,7 +1,7 @@
 #ifndef MPV_IMG_FOURCC_H
 #define MPV_IMG_FOURCC_H
 
-#include <sys/types.h>
+#include "osdep/endian.h"
 
 #define MP_FOURCC(a,b,c,d) ((a) | ((b)<<8) | ((c)<<16) | ((unsigned)(d)<<24))
 

waftools/detections/compiler.py

@@ -44,9 +44,6 @@ def __add_clang_flags__(ctx):
 
 def __add_mingw_flags__(ctx):
     ctx.env.CFLAGS += ['-D__USE_MINGW_ANSI_STDIO=1']
-    ctx.env.CFLAGS += ['-DBYTE_ORDER=1234']
-    ctx.env.CFLAGS += ['-DLITLE_ENDIAN=1234']
-    ctx.env.CFLAGS += ['-DBIG_ENDIAN=4321']
     ctx.env.LAST_LINKFLAGS += ['-mwindows']
 
 def __add_cygwin_flags__(ctx):