The following code helps in implementing this.

Note: This code only works for PDF documents. For office documents look at the next article.

<script type="text/javascript" src="http://ajax.aspnetcdn.com/ajax/jQuery/jquery-3.2.1.min.js"></script>
<script type="text/javascript">
    $("a[href$='.pdf']").each(function() {
        $(this).removeAttr("onclick").attr("href", "javascript:OpenDocumentInDialogue(\"" + this.href + "\")");
    });

    function OpenDocumentInDialogue(href) {
        var o;
        $("body").append("<div id='docTemp' style='min-height:600px;min-width:800px;height:100%;overflow:hidden'><object data='" + href + "' width='100%' height='100%' type='application/pdf' ></object></div>");
        o = {};
        o.html = $("#docTemp")[0];
        o.showMaximized = true;
        o.title = href.substring(href.lastIndexOf("/") + 1);
        o.dialogReturnValueCallback = OpenDocumentInDialogueCallback;
        SP.UI.ModalDialog.showModalDialog(o);
    }

    function OpenDocumentInDialogueCallback() {
        // Do something here when the dialog closes
    }
</script>

The post Open SharePoint PDF documents in modal dialog appeared first on Binary Bits.