rtmp: Move RTMP digest calculation to a separate file

The rtmpcrypt protocol requires it.

libavformat/Makefile

@@ -386,7 +386,7 @@ OBJS-$(CONFIG_LIBRTMP)                   += librtmp.o
 OBJS-$(CONFIG_APPLEHTTP_PROTOCOL)        += hlsproto.o
 OBJS-$(CONFIG_CONCAT_PROTOCOL)           += concat.o
 OBJS-$(CONFIG_CRYPTO_PROTOCOL)           += crypto.o
-OBJS-$(CONFIG_FFRTMPCRYPT_PROTOCOL)      += rtmpcrypt.o rtmpdh.o
+OBJS-$(CONFIG_FFRTMPCRYPT_PROTOCOL)      += rtmpcrypt.o rtmpdigest.o rtmpdh.o
 OBJS-$(CONFIG_FFRTMPHTTP_PROTOCOL)       += rtmphttp.o
 OBJS-$(CONFIG_FILE_PROTOCOL)             += file.o
 OBJS-$(CONFIG_GOPHER_PROTOCOL)           += gopher.o
@@ -399,12 +399,12 @@ OBJS-$(CONFIG_MD5_PROTOCOL)              += md5proto.o
 OBJS-$(CONFIG_MMSH_PROTOCOL)             += mmsh.o mms.o asf.o
 OBJS-$(CONFIG_MMST_PROTOCOL)             += mmst.o mms.o asf.o
 OBJS-$(CONFIG_PIPE_PROTOCOL)             += file.o
-OBJS-$(CONFIG_RTMP_PROTOCOL)             += rtmpproto.o rtmppkt.o
+OBJS-$(CONFIG_RTMP_PROTOCOL)             += rtmpproto.o rtmpdigest.o rtmppkt.o
-OBJS-$(CONFIG_RTMPE_PROTOCOL)            += rtmpproto.o rtmppkt.o
+OBJS-$(CONFIG_RTMPE_PROTOCOL)            += rtmpproto.o rtmpdigest.o rtmppkt.o
-OBJS-$(CONFIG_RTMPS_PROTOCOL)            += rtmpproto.o rtmppkt.o
+OBJS-$(CONFIG_RTMPS_PROTOCOL)            += rtmpproto.o rtmpdigest.o rtmppkt.o
-OBJS-$(CONFIG_RTMPT_PROTOCOL)            += rtmpproto.o rtmppkt.o
+OBJS-$(CONFIG_RTMPT_PROTOCOL)            += rtmpproto.o rtmpdigest.o rtmppkt.o
-OBJS-$(CONFIG_RTMPTE_PROTOCOL)           += rtmpproto.o rtmppkt.o
+OBJS-$(CONFIG_RTMPTE_PROTOCOL)           += rtmpproto.o rtmpdigest.o rtmppkt.o
-OBJS-$(CONFIG_RTMPTS_PROTOCOL)           += rtmpproto.o rtmppkt.o
+OBJS-$(CONFIG_RTMPTS_PROTOCOL)           += rtmpproto.o rtmpdigest.o rtmppkt.o
 OBJS-$(CONFIG_RTP_PROTOCOL)              += rtpproto.o
 OBJS-$(CONFIG_SCTP_PROTOCOL)             += sctp.o
 OBJS-$(CONFIG_SRTP_PROTOCOL)             += srtpproto.o srtp.o

libavformat/rtmpdigest.c

@@ -0,0 +1,67 @@
+/*
+ * RTMP network protocol
+ * Copyright (c) 2009 Konstantin Shishkov
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * RTMP protocol digest
+ */
+
+#include <stdint.h>
+
+#include "libavutil/error.h"
+#include "libavutil/hmac.h"
+
+#include "rtmp.h"
+
+int ff_rtmp_calc_digest(const uint8_t *src, int len, int gap,
+                        const uint8_t *key, int keylen, uint8_t *dst)
+{
+    AVHMAC *hmac;
+
+    hmac = av_hmac_alloc(AV_HMAC_SHA256);
+    if (!hmac)
+        return AVERROR(ENOMEM);
+
+    av_hmac_init(hmac, key, keylen);
+    if (gap <= 0) {
+        av_hmac_update(hmac, src, len);
+    } else { //skip 32 bytes used for storing digest
+        av_hmac_update(hmac, src, gap);
+        av_hmac_update(hmac, src + gap + 32, len - gap - 32);
+    }
+    av_hmac_final(hmac, dst, 32);
+
+    av_hmac_free(hmac);
+
+    return 0;
+}
+
+int ff_rtmp_calc_digest_pos(const uint8_t *buf, int off, int mod_val,
+                            int add_val)
+{
+    int i, digest_pos = 0;
+
+    for (i = 0; i < 4; i++)
+        digest_pos += buf[i + off];
+    digest_pos = digest_pos % mod_val + add_val;
+
+    return digest_pos;
+}

libavformat/rtmpproto.c

@@ -27,7 +27,6 @@
 #include "libavcodec/bytestream.h"
 #include "libavutil/avstring.h"
 #include "libavutil/base64.h"
-#include "libavutil/hmac.h"
 #include "libavutil/intfloat.h"
 #include "libavutil/lfg.h"
 #include "libavutil/md5.h"
@@ -957,41 +956,6 @@ static int gen_fcsubscribe_stream(URLContext *s, RTMPContext *rt,
     return rtmp_send_packet(rt, &pkt, 1);
 }
 
-int ff_rtmp_calc_digest(const uint8_t *src, int len, int gap,
-                        const uint8_t *key, int keylen, uint8_t *dst)
-{
-    AVHMAC *hmac;
-
-    hmac = av_hmac_alloc(AV_HMAC_SHA256);
-    if (!hmac)
-        return AVERROR(ENOMEM);
-
-    av_hmac_init(hmac, key, keylen);
-    if (gap <= 0) {
-        av_hmac_update(hmac, src, len);
-    } else { //skip 32 bytes used for storing digest
-        av_hmac_update(hmac, src, gap);
-        av_hmac_update(hmac, src + gap + 32, len - gap - 32);
-    }
-    av_hmac_final(hmac, dst, 32);
-
-    av_hmac_free(hmac);
-
-    return 0;
-}
-
-int ff_rtmp_calc_digest_pos(const uint8_t *buf, int off, int mod_val,
-                            int add_val)
-{
-    int i, digest_pos = 0;
-
-    for (i = 0; i < 4; i++)
-        digest_pos += buf[i + off];
-    digest_pos = digest_pos % mod_val + add_val;
-
-    return digest_pos;
-}
-
 /**
  * Put HMAC-SHA2 digest of packet data (except for the bytes where this digest
  * will be stored) into that packet.