The solution I would suggest is to blow away the advimage button and specify your own upload image button. good for if you are using the jquery plugin.
$('textarea.tinymce').tinymce({
// Location of TinyMCE script
script_url : '../../../js/vendor/tiny_mce/tiny_mce.js',
// General options
theme : "advanced",
skin : "bootstrap",
plugins : "pagebreak,style,layer,table,save,advhr,advimage,advlink,emotions,iespell,inlinepopups,insertdatetime,preview,media,searchreplace,print,contextmenu,paste,directionality,fullscreen,noneditable,visualchars,nonbreaking,xhtmlxtras,template",
// Theme options
theme_advanced_buttons1 : "uploadimageCustom, save,newdocument,|,bold,italic,underline,strikethrough,|,justifyleft,justifycenter,justifyright,justifyfull,styleselect,formatselect,fontselect,fontsizeselect",
theme_advanced_toolbar_location : "top",
theme_advanced_toolbar_align : "left",
theme_advanced_statusbar_location : "bottom",
theme_advanced_resizing : true,
// Drop lists for link/image/media/template dialogs
template_external_list_url : "lists/template_list.js",
external_link_list_url : "lists/link_list.js",
external_image_list_url : "lists/image_list.js",
media_external_list_url : "lists/media_list.js",
// Replace values for the template plugin
template_replace_values : {
username : "Some User",
staffid : "991234"
},
setup: function (fn) {
fn.addButton('uploadimageCustom', {
title : 'Upload Image',
onclick : function () {
// pop open your own upload modal.
$(selector).modal('show')
// modal has fine uploader or other plugin.
// after xhr for image uploading is done or on 'complete' event for fineuploader
// insert your image html to the textarea with
$('#content').tinymce().execCommand('mceInsertContent',false,'<img src="<your newly uploaded image src>" />');"
}
});
}