Download chat history with remote JID

backend/whatsapp.py

@@ -4,6 +4,7 @@
 import sys;
 sys.dont_write_bytecode = True;
 
+import binascii;
 import os;
 import signal;
 import base64;
@@ -24,13 +25,21 @@
 import websocket;
 import curve25519;
 import pyqrcode;
+
+from Crypto import Random;
 from utilities import *;
 from whatsapp_binary_reader import whatsappReadBinary;
+from whatsapp_binary_writer import whatsappWriteBinary;
+from whatsapp_defines import WAMetrics;
 
 reload(sys);
 sys.setdefaultencoding("utf-8");
 
 
+def is_callback_valid_in(pend):
+    return "callback" in pend and pend["callback"] is not None and\
+           "func" in pend["callback"] and pend["callback"]["func"] is not None and\
+           "tag" in pend["callback"] and pend["callback"]["tag"] is not None
 
 def HmacSha256(key, sign):
     return hmac.new(key, sign, hashlib.sha256).digest();
@@ -132,11 +141,12 @@ def onClose(self, ws):
         eprint("WhatsApp backend Websocket closed.");
 
     def onMessage(self, ws, message):
+
         try:
             messageSplit = message.split(",", 1);
             messageTag = messageSplit[0];
             messageContent = messageSplit[1];
-            
+
             if messageTag in self.messageQueue:											# when the server responds to a client's message
                 pend = self.messageQueue[messageTag];
                 if pend["desc"] == "_status":
@@ -155,26 +165,24 @@ def onMessage(self, ws, message):
 
                     svgBuffer = io.BytesIO();											# from https://github.com/mnooner256/pyqrcode/issues/39#issuecomment-207621532
                     pyqrcode.create(qrCodeContents, error='L').svg(svgBuffer, scale=6, background="rgba(0,0,0,0.0)", module_color="#122E31", quiet_zone=0);
-                    if "callback" in pend and pend["callback"] is not None and "func" in pend["callback"] and pend["callback"]["func"] is not None and "tag" in pend["callback"] and pend["callback"]["tag"] is not None:
-                        pend["callback"]["func"]({ "type": "generated_qr_code", "image": "data:image/svg+xml;base64," + base64.b64encode(svgBuffer.getvalue()), "content": qrCodeContents }, pend["callback"]);
+                    if is_callback_valid_in(pend):
+                        pend["callback"]["func"]({"type": "generated_qr_code", "image": "data:image/svg+xml;base64," + base64.b64encode(svgBuffer.getvalue()), "content": qrCodeContents }, pend["callback"]);
+                elif pend["desc"] == "_chatHistory":
+                    if messageContent != "":
+                        messageDecrypted = self.decrypt_message_content(messageContent);
+                        if is_callback_valid_in(pend):
+                            pend["callback"]["func"]({
+                                "type": "chat_history",
+                                "jid": pend["callback"]["args"]["jid"],
+                                "content": messageDecrypted
+                            }, pend["callback"]);
             else:
                 try:
                     jsonObj = json.loads(messageContent);								# try reading as json
-                except ValueError, e:
+                except ValueError:
                     if messageContent != "":
-                        hmacValidation = HmacSha256(self.loginInfo["key"]["macKey"], messageContent[32:]);
-                        if hmacValidation != messageContent[:32]:
-                            raise ValueError("Hmac mismatch");
-
-                        decryptedMessage = AESDecrypt(self.loginInfo["key"]["encKey"], messageContent[32:]);
-                        try:
-                            processedData = whatsappReadBinary(decryptedMessage, True);
-                            messageType = "binary";
-                        except:
-                            processedData = { "traceback": traceback.format_exc().splitlines() };
-                            messageType = "error";
-                        finally:
-                            self.onMessageCallback["func"](processedData, self.onMessageCallback, { "message_type": messageType });
+                        processedData, messageType = self.decrypt_message_content(messageContent)
+                        self.onMessageCallback["func"](processedData, self.onMessageCallback, {"message_type": messageType});
                 else:
                     self.onMessageCallback["func"](jsonObj, self.onMessageCallback, { "message_type": "json" });
                     if isinstance(jsonObj, list) and len(jsonObj) > 0:					# check if the result is an array
@@ -185,7 +193,7 @@ def onMessage(self, ws, message):
                             self.connInfo["serverToken"] = jsonObj[1]["serverToken"];
                             self.connInfo["browserToken"] = jsonObj[1]["browserToken"];
                             self.connInfo["me"] = jsonObj[1]["wid"];
-                            
+
                             self.connInfo["secret"] = base64.b64decode(jsonObj[1]["secret"]);
                             self.connInfo["sharedSecret"] = self.loginInfo["privateKey"].get_shared_key(curve25519.Public(self.connInfo["secret"][:32]), lambda a: a);
                             sse = self.connInfo["sharedSecretExpanded"] = HKDF(self.connInfo["sharedSecret"], 80);
@@ -197,7 +205,7 @@ def onMessage(self, ws, message):
                             keysDecrypted = AESDecrypt(sse[:32], keysEncrypted);
                             self.loginInfo["key"]["encKey"] = keysDecrypted[:32];
                             self.loginInfo["key"]["macKey"] = keysDecrypted[32:64];
-                            
+
                             # eprint("private key            : ", base64.b64encode(self.loginInfo["privateKey"].serialize()));
                             # eprint("secret                 : ", base64.b64encode(self.connInfo["secret"]));
                             # eprint("shared secret          : ", base64.b64encode(self.connInfo["sharedSecret"]));
@@ -215,7 +223,19 @@ def onMessage(self, ws, message):
         except:
             eprint(traceback.format_exc());
 
-
+    def decrypt_message_content(self, messageContent):
+        hmacValidation = HmacSha256(self.loginInfo["key"]["macKey"], messageContent[32:]);
+        if hmacValidation != messageContent[:32]:
+            raise ValueError("Hmac mismatch");
+        decryptedMessage = AESDecrypt(self.loginInfo["key"]["encKey"], messageContent[32:]);
+        try:
+            processedData = whatsappReadBinary(decryptedMessage, True);
+            messageType = "binary";
+        except:
+            processedData = {"traceback": traceback.format_exc().splitlines()};
+            messageType = "error";
+        finally:
+            return processedData, messageType
 
     def connect(self):
         self.activeWs = websocket.WebSocketApp("wss://web.whatsapp.com/ws",
@@ -224,7 +244,7 @@ def connect(self):
                                                on_open = lambda ws: self.onOpen(ws),
                                                on_close = lambda ws: self.onClose(ws),
                                                header = { "Origin: https://web.whatsapp.com" });
-        
+
         self.websocketThread = Thread(target = self.activeWs.run_forever);
         self.websocketThread.daemon = True;
         self.websocketThread.start();
@@ -235,7 +255,7 @@ def generateQRCode(self, callback=None):
         self.messageQueue[messageTag] = { "desc": "_login", "callback": callback };
         message = messageTag + ',["admin","init",[0,3,2390],["Chromium at ' + datetime.datetime.now().isoformat() + '","Chromium"],"' + self.loginInfo["clientId"] + '",true]';
         self.activeWs.send(message);
-        
+
     def restoreSession(self, callback=None):
         messageTag = str(getTimestamp())
         message = messageTag + ',["admin","init",[0,3,2390],["Chromium at ' + datetime.now().isoformat() + '","Chromium"],"' + self.loginInfo["clientId"] + '",true]'
@@ -247,13 +267,42 @@ def restoreSession(self, callback=None):
             "serverToken"] + '", "' + self.loginInfo["clientId"] + '", "takeover"]'
 
         self.activeWs.send(message)
-        
+
     def getLoginInfo(self, callback):
         callback["func"]({ "type": "login_info", "data": self.loginInfo }, callback);
-    
+
     def getConnectionInfo(self, callback):
         callback["func"]({ "type": "connection_info", "data": self.connInfo }, callback);
-
+
+    def get_chat_history(self, callback, jid, count=10000):
+        """
+
+        :param callback:
+        :param jid:
+        :param count:
+        :return:
+        """
+
+        messageTag = "3EB0"+binascii.hexlify(Random.get_random_bytes(8)).upper()
+        self.messageQueue[messageTag] = { "desc": "_chatHistory", "callback": callback};
+        messageTag = self.__send_request(messageTag,
+            ["query", {"type": "message", "jid": jid, "count": str(count)}, None],
+            WAMetrics.QUERY_MESSAGES
+        );
+
+
+    def __send_request(self, messageTag, msgData, metrics):
+        encryptedMessage = WhatsAppEncrypt(
+            self.loginInfo["key"]["encKey"],
+            self.loginInfo["key"]["macKey"],
+            whatsappWriteBinary(msgData)
+        )
+
+        payload = bytearray(messageTag) + bytearray(",") + bytearray(
+            to_bytes(metrics, 1)
+        ) + bytearray([0x80]) + encryptedMessage
+        self.activeWs.send(payload, websocket.ABNF.OPCODE_BINARY)
+
     def sendTextMessage(self, number, text):
         messageId = "3EB0"+binascii.hexlify(Random.get_random_bytes(8)).upper()
         messageTag = str(getTimestamp())
@@ -264,7 +313,7 @@ def sendTextMessage(self, number, text):
         self.messageSentCount = self.messageSentCount + 1
         self.messageQueue[messageId] = {"desc": "__sending"}
         self.activeWs.send(payload, websocket.ABNF.OPCODE_BINARY)
-        
+
     def status(self, callback=None):
         if self.activeWs is not None:
             messageTag = str(getTimestamp())

backend/whatsapp_web_backend.py

@@ -3,6 +3,8 @@
 
 from __future__ import print_function;
 import sys;
+
+
 sys.dont_write_bytecode = True;
 
 import os;
@@ -75,10 +77,14 @@ def handleMessage(self):
                     self.clientInstances[clientInstanceId] = WhatsAppWebClient(onOpenCallback, onMessageCallback, onCloseCallback);
                 else:
                     currWhatsAppInstance = self.clientInstances[obj["whatsapp_instance_id"]];
+
+                    def callback_function(obj_, cbSelf_):
+                        self.sendJSON(mergeDicts(obj_, getAttr(cbSelf_, "args")), getAttr(cbSelf_, "tag"))
+
                     callback = {
-                        "func": lambda obj, cbSelf: self.sendJSON(mergeDicts(obj, getAttr(cbSelf, "args")), getAttr(cbSelf, "tag")),
+                        "func": callback_function,
                         "tag": tag,
-                        "args": { "resource_instance_id": obj["whatsapp_instance_id"] }
+                        "args": {"resource_instance_id": obj["whatsapp_instance_id"]}
                     };
                     if currWhatsAppInstance.activeWs is None:
                         self.sendError("No WhatsApp server connected to backend.");
@@ -91,6 +97,12 @@ def handleMessage(self):
                         currWhatsAppInstance.getLoginInfo(callback);
                     elif cmd == "backend-getConnectionInfo":
                         currWhatsAppInstance.getConnectionInfo(callback);
+                    elif cmd == "backend-getChatHistory":
+                        currWhatsAppInstance.get_chat_history({
+                            "func": callback_function,
+                            "tag": tag,
+                            "args": {"resource_instance_id": obj["whatsapp_instance_id"], "jid": obj["jid"]}
+                        }, str(obj["jid"]));
                     elif cmd == "backend-disconnectWhatsApp":
                         currWhatsAppInstance.disconnect();
                         self.sendJSON({ "type": "resource_disconnected", "resource": "whatsapp", "resource_instance_id": obj["whatsapp_instance_id"] }, tag);

client/index.html

@@ -3,9 +3,9 @@
         <meta charset="utf-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge">
         <meta name="viewport" content="width=device-width, initial-scale=1">
-        
+
         <title>WhatsApp Web</title>
-        
+
         <!-- stylesheets -->
         <link rel="stylesheet" href="lib/bootstrap/css/bootstrap.min.css">
         <link rel="stylesheet" href="lib/font-awesome/css/font-awesome.min.css">
@@ -62,7 +62,16 @@ <h1>WhatsApp Web</h1>
             <button id="console-arrow-button" class="btn"><i class="fa fa-angle-left" aria-hidden="true"></i></button>
         </div>
         <div id="console">
-            Hello
+            <form class="form-inline hidden" id="formDlChat">
+                <span>Download chat history (.json)</span>
+                <div class="form-group">
+                    <label for="chatDlRemoteJID">Chat JID</label>
+                    <input type="text" class="form-control" id="chatDlRemoteJID" placeholder="Enter chat JID">
+                </div>
+                <button type="submit" class="btn btn-default" value="download">
+                    <i class="fa fa-download" aria-hidden="true"></i>
+                </button>
+            </form>
         </div>
     </body>
 </html>

client/js/main.js

@@ -6,8 +6,44 @@ function sleep(ms) {
         setTimeout(() => resolve(), ms);
     });
 }
+function request_chat_history(jid) {
+    new BootstrapStep({
+        websocket: apiWebsocket,
+        request: {
+            type: "call",
+            callArgs: { command: "backend-getChatHistory", jid: jid },
+            successCondition: obj => "jid" in obj && "type" in obj &&
+                obj.jid === jid && obj.type === "chat_history" && "messages" in obj && Array.isArray(obj.messages),
+            successActor: (websocket, {messages, jid})=>  {
+                download("messages-" + jid + ".json", JSON.stringify(messages));
+            }
+        }
+    }).run().catch(() => currentRequestJID = null);
+}
+/**
+ * https://stackoverflow.com/questions/3665115/how-to-create-a-file-in-memory-for-user-to-download-but-not-through-server
+ * @param filename
+ * @param text
+ */
+function download(filename, text) {
+    var element = document.createElement('a');
+    element.setAttribute('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(text));
+    element.setAttribute('download', filename);
+
+    element.style.display = 'none';
+    document.body.appendChild(element);
+
+    element.click();
+
+    document.body.removeChild(element);
+}
 
 $(document).ready(function() {
+    $("#formDlChat").submit((event) => {
+        event.preventDefault();
+        request_chat_history($("#chatDlRemoteJID").val());
+    });
+
     $("#console-arrow-button").click(() => {
         if(consoleShown) {
             $("#console-arrow").removeClass("extended").find("i.fa").removeClass("fa-angle-right").addClass("fa-angle-left");
@@ -19,10 +55,10 @@ $(document).ready(function() {
         }
         consoleShown = !consoleShown;
     });
-    
+
     const responseTimeout = 10000;
     let bootstrapState = 0;
-    
+
 
 
     let apiInfo = {
@@ -40,12 +76,16 @@ $(document).ready(function() {
 
     let allWhatsAppMessages = [];
     let bootstrapInfo = {
+        deactivated: false,
         activateButton: (text, buttonEnabled) => {
             let container = $("#bootstrap-container").removeClass("hidden").children("#bootstrap-container-content");
             container.children("img").detach();
             container.children("button").removeClass("hidden").html(text).attr("disabled", !buttonEnabled);
             $("#main-container").addClass("hidden");
 
+            $("#formDlChat").addClass("hidden");
+            this.deactivated = false;
+
             allWhatsAppMessages = [];
             $("#messages-list-table-body").empty();
         },
@@ -56,9 +96,14 @@ $(document).ready(function() {
             $("#main-container").addClass("hidden");
         },
         deactivate: () => {
-            $("#bootstrap-container").addClass("hidden");
+            if (this.deactivated) return;
+            this.deactivated = true;
+            $("#bootstrap-container").addClass("hidden")
+
+            $("#formDlChat").removeClass("hidden");
             $("#main-container").removeClass("hidden");
             $("#button-disconnect").html("Disconnect").attr("disabled", false);
+
         },
         steps: [
             new BootstrapStep({
@@ -152,6 +197,7 @@ $(document).ready(function() {
                             keepWhenHit: true
                         }).then(whatsAppMessage => {
                             bootstrapInfo.deactivate();
+
                             /*<tr>
                                 <th scope="row">1</th>
                                 <td>Do., 21.12.2017, 22:59:09.123</td>
@@ -191,7 +237,7 @@ $(document).ready(function() {
                                     tree = jsonTree.create(jsonData, dialog.find(".bootbox-body").empty()[0]);
                                 });
                             });
-                            
+
                             let tableRow = $("<tr></tr>").attr("data-message-index", allWhatsAppMessages.length);
                             tableRow.append($("<th></th>").attr("scope", "row").html(allWhatsAppMessages.length+1));
                             tableRow.append($("<td></td>").html(moment.unix(d.timestamp/1000.0).format("ddd, DD.MM.YYYY, HH:mm:ss.SSS")));
@@ -229,7 +275,7 @@ $(document).ready(function() {
     $("#button-disconnect").click(function() {
         if(!apiWebsocket.backendConnectedToWhatsApp)
             return;
-        
+
         $(this).attr("disabled", true).html("Disconnecting...");
         new BootstrapStep({
             websocket: apiWebsocket,